gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-16 10:28:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: Major event processing system refactor for improved performance

Browse Source
This commit is contained in:
ysq
2025-08-27 21:31:31 +08:00
parent e673b0aab8
commit 9ea4dab4df
22 changed files with 829 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/common/event_processor.rs
+71 -73
View File
@@ -1,7 +1,7 @@
use std::sync::Arc;
use tokio::task::JoinHandle;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::Signature;
use crate::common::AnyResult;
use crate::streaming::common::{
@@ -14,7 +14,7 @@ use crate::streaming::event_parser::EventParser;
use crate::streaming::event_parser::{
core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
};
use crate::streaming::grpc::{BackpressureStrategy, BatchConfig, EventPretty};
use crate::streaming::grpc::{BackpressureConfig, BatchConfig, EventPretty};
use crate::streaming::shred::TransactionWithSlot;
use once_cell::sync::OnceCell;
@@ -25,21 +25,24 @@ pub struct EventProcessor {
pub(crate) parser_cache: OnceCell<Arc<dyn EventParser>>,
pub(crate) protocols: Vec<Protocol>,
pub(crate) event_type_filter: Option<EventTypeFilter>,
pub(crate) backpressure_strategy: BackpressureStrategy,
pub(crate) callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
pub(crate) backpressure_config: BackpressureConfig,
pub(crate) batch_config: BatchConfig,
}
impl EventProcessor {
/// 创建新的事件处理器
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
let backpressure_config = config.backpressure.clone();
Self {
metrics_manager,
config,
parser_cache: OnceCell::new(),
protocols: vec![],
event_type_filter: None,
backpressure_strategy: BackpressureStrategy::Block,
backpressure_config,
batch_config: BatchConfig::default(),
callback: None,
}
}
@@ -47,13 +50,15 @@ impl EventProcessor {
&mut self,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
backpressure_strategy: BackpressureStrategy,
backpressure_config: BackpressureConfig,
batch_config: BatchConfig,
callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
) {
self.protocols = protocols.clone();
self.event_type_filter = event_type_filter.clone();
self.backpressure_strategy = backpressure_strategy;
self.backpressure_config = backpressure_config;
self.batch_config = batch_config;
self.callback = callback;
self.parser_cache
.get_or_init(|| Arc::new(MutilEventParser::new(protocols, event_type_filter)));
}
@@ -62,35 +67,24 @@ impl EventProcessor {
self.parser_cache.get().unwrap().clone()
}
pub fn get_event_handle(&self) -> Option<JoinHandle<()>> {
return None;
}
pub async fn process_grpc_event_transaction_with_metrics<F>(
pub async fn process_grpc_event_transaction_with_metrics(
&self,
event_pretty: EventPretty,
callback: &F,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
self.process_grpc_event_transaction(event_pretty, callback, bot_wallet).await?;
) -> AnyResult<()> {
self.process_grpc_event_transaction(event_pretty, bot_wallet).await?;
Ok(())
}
async fn process_grpc_event_transaction<F>(
async fn process_grpc_event_transaction(
&self,
event_pretty: EventPretty,
callback: &F,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
) -> AnyResult<()> {
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count();
let signature = account_pretty.signature;
let account_event = AccountEventParser::parse_account_event(
self.protocols.clone(),
account_pretty,
@@ -98,12 +92,12 @@ impl EventProcessor {
);
if let Some(event) = account_event {
let processing_time_us = event.program_handle_time_consuming_us() as f64;
callback(event);
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(
self.invoke_callback(event);
self.update_metrics(
MetricsEventType::Account,
1,
processing_time_us,
Some(signature),
);
}
}
@@ -113,7 +107,7 @@ impl EventProcessor {
let signature = transaction_pretty.signature;
// 使用缓存获取解析器
let parser = self.get_parser();
let mut all_events = parser
let all_events = parser
.parse_transaction(
transaction_pretty.tx.clone(),
signature,
@@ -125,37 +119,26 @@ impl EventProcessor {
.await
.unwrap_or_else(|_e| vec![]);
// 为所有事件设置交易索引
for event in &mut all_events {
event.set_transaction_index(transaction_pretty.transaction_index);
}
let max_time_consuming_us = all_events
.iter()
.map(|event| event.program_handle_time_consuming_us())
.max()
.unwrap_or(0);
// 保存事件数量用于日志记录
let mut max_time_consuming_us = 0;
let event_count = all_events.len();
// 批量处理事件
if !all_events.is_empty() {
for mut event in all_events {
event.set_program_handle_time_consuming_us(
chrono::Utc::now().timestamp_micros()
- event.program_received_time_us(),
);
callback(event);
}
// 为所有事件设置交易索引
for mut event in all_events {
event.set_transaction_index(transaction_pretty.transaction_index);
event.set_program_handle_time_consuming_us(
chrono::Utc::now().timestamp_micros() - event.program_received_time_us(),
);
max_time_consuming_us =
max_time_consuming_us.max(event.program_handle_time_consuming_us());
self.invoke_callback(event);
}
// 更新性能指标
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(
MetricsEventType::Tx,
self.update_metrics(
MetricsEventType::Transaction,
event_count as u64,
max_time_consuming_us as f64,
Some(signature),
);
}
EventPretty::BlockMeta(block_meta_pretty) => {
@@ -171,29 +154,34 @@ impl EventProcessor {
block_meta_pretty.program_received_time_us,
);
let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64;
callback(block_meta_event);
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(
MetricsEventType::BlockMeta,
1,
processing_time_us,
);
self.invoke_callback(block_meta_event);
self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us, None);
}
}
Ok(())
}
pub fn invoke_callback(&self, event: Box<dyn UnifiedEvent>) {
if let Some(callback) = self.callback.as_ref() {
callback(event);
}
}
/// 即时处理单个交易
pub async fn process_shred_transaction_immediate<F>(
pub async fn process_shred_transaction_immediate(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
callback: &F,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
) -> AnyResult<()> {
self.process_shred_transaction(transaction_with_slot, bot_wallet).await
}
pub async fn process_shred_transaction(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
self.metrics_manager.add_tx_process_count();
let program_received_time_us = chrono::Utc::now().timestamp_micros();
let slot = transaction_with_slot.slot;
@@ -215,11 +203,7 @@ impl EventProcessor {
.await
.unwrap_or_else(|_e| vec![]);
let max_time_consuming_us = all_events
.iter()
.map(|event| event.program_handle_time_consuming_us())
.max()
.unwrap_or(0);
let mut max_time_consuming_us = 0;
// 保存事件数量用于日志记录
let event_count = all_events.len();
@@ -229,18 +213,31 @@ impl EventProcessor {
event.set_program_handle_time_consuming_us(
chrono::Utc::now().timestamp_micros() - event.program_received_time_us(),
);
callback(event);
max_time_consuming_us =
max_time_consuming_us.max(event.program_handle_time_consuming_us());
self.invoke_callback(event);
}
// 实际调用性能指标更新
self.metrics_manager.update_metrics(
MetricsEventType::Tx,
self.update_metrics(
MetricsEventType::Transaction,
event_count as u64,
max_time_consuming_us as f64,
Some(signature),
);
Ok(())
}
fn update_metrics(
&self,
ty: MetricsEventType,
count: u64,
time_us: f64,
signature: Option<Signature>,
) {
self.metrics_manager.update_metrics(ty, count, time_us, signature);
}
}
// 实现 Clone trait 以支持模块间共享
@@ -252,8 +249,9 @@ impl Clone for EventProcessor {
parser_cache: self.parser_cache.clone(),
protocols: self.protocols.clone(),
event_type_filter: self.event_type_filter.clone(),
backpressure_strategy: self.backpressure_strategy.clone(),
backpressure_config: self.backpressure_config.clone(),
batch_config: self.batch_config.clone(),
callback: self.callback.clone(),
}
}
}