gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-22 21:38:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: Refactor event processing system for better performance

Browse Source
This commit is contained in:
ysq
2025-08-26 17:57:39 +08:00
parent f87f3dd7f3
commit dee683396d
33 changed files with 1158 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/grpc/event_processor.rs
-290
View File
@@ -1,290 +0,0 @@
use std::sync::{Arc, Mutex};
use solana_sdk::pubkey::Pubkey;
use super::types::EventPretty;
use crate::common::AnyResult;
use crate::streaming::common::{
EventBatchProcessor as EventBatchCollector, MetricsEventType, MetricsManager,
StreamClientConfig as ClientConfig,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
use crate::streaming::event_parser::EventParser;
use crate::streaming::event_parser::{
core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol,
};
/// 事件处理器
pub struct EventProcessor {
pub(crate) metrics_manager: MetricsManager,
pub(crate) config: ClientConfig,
pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
}
impl EventProcessor {
/// 创建新的事件处理器
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
}
/// 获取或创建解析器，使用缓存机制避免重复创建
fn get_or_create_parser(
&self,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> Arc<dyn EventParser> {
let mut cache = self.parser_cache.lock().unwrap();
if let Some(cached_parser) = cache.clone() {
return cached_parser.clone();
}
let parser: Arc<dyn EventParser> =
Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
*cache = Some(parser.clone());
parser
}
/// 使用性能监控处理事件交易
pub async fn process_event_transaction_with_metrics<F>(
&self,
event_pretty: EventPretty,
callback: &F,
bot_wallet: Option<Pubkey>,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let account_event = AccountEventParser::parse_account_event(
protocols.clone(),
account_pretty,
program_received_time_ms,
event_type_filter,
);
if let Some(event) = account_event {
callback(event);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_tx_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature.to_string();
// 使用缓存获取解析器
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
let all_events = parser
.parse_transaction(
transaction_pretty.tx.clone(),
&signature,
Some(slot),
transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
seconds: ts.seconds,
nanos: ts.nanos,
}),
program_received_time_ms,
bot_wallet,
)
.await
.unwrap_or_else(|_e| vec![]);
// 保存事件数量用于日志记录
let event_count = all_events.len();
// 批量处理事件
if !all_events.is_empty() {
for event in all_events {
callback(event);
}
}
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager
.update_metrics(MetricsEventType::Tx, event_count as u64, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
}
EventPretty::BlockMeta(block_meta_pretty) => {
let start_time = std::time::Instant::now();
self.metrics_manager.add_block_meta_process_count().await;
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
let block_meta_event = CommonEventParser::generate_block_meta_event(
block_meta_pretty.slot,
&block_meta_pretty.block_hash,
block_time_ms,
);
callback(block_meta_event);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
Ok(())
}
/// 使用批处理处理事件交易
pub async fn process_event_transaction_with_batch<F>(
&self,
event_pretty: EventPretty,
batch_processor: &mut EventBatchCollector<F>,
bot_wallet: Option<Pubkey>,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: Fn(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
{
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let account_event = AccountEventParser::parse_account_event(
protocols.clone(),
account_pretty,
program_received_time_ms,
event_type_filter,
);
if let Some(event) = account_event {
(batch_processor.callback)(vec![event]);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.metrics_manager
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_tx_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature.to_string();
// 使用缓存获取解析器
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
let result = parser
.parse_transaction(
transaction_pretty.tx.clone(),
&signature,
Some(slot),
transaction_pretty.block_time.map(|ts| prost_types::Timestamp {
seconds: ts.seconds,
nanos: ts.nanos,
}),
program_received_time_ms,
bot_wallet,
)
.await;
// 处理解析结果并使用批处理器
let total_events = match result {
Ok(events) => {
let event_count = events.len();
if !events.is_empty() {
log::debug!("Parsed {} events", event_count);
log::debug!("Adding {} events to batch processor", event_count);
for event in events {
if self.config.batch.enabled {
batch_processor.add_event(event);
} else {
// 如果批处理被禁用，直接调用回调
// 这里需要将单个事件包装成Vec来调用批处理回调
let single_event_batch = vec![event];
(batch_processor.callback)(single_event_batch);
}
}
}
event_count
}
Err(e) => {
log::warn!("Failed to parse transaction: {:?}", e);
0
}
};
// 添加调试信息
if total_events > 0 {
log::debug!(
"Total events parsed: {} for transaction {}",
total_events,
signature
);
}
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.metrics_manager
.update_metrics(MetricsEventType::Tx, total_events as u64, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, total_events);
}
EventPretty::BlockMeta(block_meta_pretty) => {
let start_time = std::time::Instant::now();
self.metrics_manager.add_block_meta_process_count().await;
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
let block_meta_event = CommonEventParser::generate_block_meta_event(
block_meta_pretty.slot,
&block_meta_pretty.block_hash,
block_time_ms,
);
(batch_processor.callback)(vec![block_meta_event]);
// 更新性能指标
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
Ok(())
}
}
src/streaming/grpc/mod.rs
+7 -15
View File
@@ -1,25 +1,17 @@
// gRPC 相关模块
pub mod connection;
pub mod types;
pub mod subscription;
pub mod stream_handler;
pub mod event_processor;
pub mod subscription;
pub mod types;
// 重新导出主要类型
pub use connection::*;
pub use types::*;
pub use subscription::*;
pub use stream_handler::*;
pub use event_processor::*;
pub use subscription::*;
pub use types::*;
// 从公用模块重新导出
pub use crate::streaming::common::{
StreamClientConfig as ClientConfig,
PerformanceMetrics,
MetricsManager,
EventBatchProcessor as EventBatchCollector,
BackpressureStrategy,
BatchConfig,
BackpressureConfig,
ConnectionConfig,
};
BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, MetricsManager,
PerformanceMetrics, StreamClientConfig as ClientConfig,
};
src/streaming/grpc/stream_handler.rs
+86 -79
View File
@@ -1,12 +1,14 @@
use chrono::Local;
use futures::{channel::mpsc, sink::Sink, SinkExt};
use solana_sdk::pubkey::Pubkey;
use yellowstone_grpc_proto::geyser::{
subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate,
};
use super::types::{BlockMetaPretty, EventPretty, TransactionPretty};
use crate::common::AnyResult;
use crate::streaming::common::BackpressureStrategy;
use crate::streaming::common::EventProcessor;
use crate::streaming::event_parser::UnifiedEvent;
use crate::streaming::grpc::AccountPretty;
/// 流消息处理器
@@ -14,33 +16,39 @@ pub struct StreamHandler;
impl StreamHandler {
/// 处理单个流消息
pub async fn handle_stream_message(
pub async fn handle_stream_message<F>(
msg: SubscribeUpdate,
tx: &mut mpsc::Sender<EventPretty>,
subscribe_tx: &mut (impl Sink<SubscribeRequest, Error = mpsc::SendError> + Unpin),
backpressure_strategy: BackpressureStrategy,
) -> AnyResult<()> {
event_processor: EventProcessor,
callback: &F,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
let created_at = msg.created_at;
match msg.update_oneof {
Some(UpdateOneof::Account(account)) => {
let account_pretty = AccountPretty::from(account);
log::debug!("Received account: {:?}", account_pretty);
Self::handle_backpressure(
tx,
EventPretty::Account(account_pretty),
backpressure_strategy,
)
.await?;
event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::Account(account_pretty),
callback,
bot_wallet,
)
.await?;
}
Some(UpdateOneof::BlockMeta(sut)) => {
let block_meta_pretty = BlockMetaPretty::from((sut, created_at));
log::debug!("Received block meta: {:?}", block_meta_pretty);
Self::handle_backpressure(
tx,
EventPretty::BlockMeta(block_meta_pretty),
backpressure_strategy,
)
.await?;
event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::BlockMeta(block_meta_pretty),
callback,
bot_wallet,
)
.await?;
}
Some(UpdateOneof::Transaction(sut)) => {
let transaction_pretty = TransactionPretty::from((sut, created_at));
@@ -49,14 +57,13 @@ impl StreamHandler {
transaction_pretty.signature,
transaction_pretty.slot
);
// 根据背压策略处理发送
Self::handle_backpressure(
tx,
EventPretty::Transaction(transaction_pretty),
backpressure_strategy,
)
.await?;
event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::Transaction(transaction_pretty),
callback,
bot_wallet,
)
.await?;
}
Some(UpdateOneof::Ping(_)) => {
subscribe_tx
@@ -77,58 +84,58 @@ impl StreamHandler {
Ok(())
}
/// 处理背压策略
async fn handle_backpressure(
tx: &mut mpsc::Sender<EventPretty>,
event_pretty: EventPretty,
backpressure_strategy: BackpressureStrategy,
) -> AnyResult<()> {
match backpressure_strategy {
BackpressureStrategy::Block => {
// 阻塞等待，直到有空间
if let Err(e) = tx.send(event_pretty).await {
log::error!("Failed to send transaction to channel: {:?}", e);
return Err(anyhow::anyhow!("Channel send failed: {:?}", e));
}
}
BackpressureStrategy::Drop => {
// 尝试发送，如果失败则丢弃
if let Err(e) = tx.try_send(event_pretty) {
if e.is_full() {
log::warn!("Channel is full, dropping transaction");
} else {
log::error!("Channel is closed: {:?}", e);
return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
}
}
}
BackpressureStrategy::Retry { max_attempts, wait_ms } => {
// 重试有限次数
let mut retry_count = 0;
loop {
match tx.try_send(event_pretty.clone()) {
Ok(_) => break,
Err(e) => {
if e.is_full() {
retry_count += 1;
if retry_count >= max_attempts {
log::warn!(
"Channel is full after {} attempts, dropping transaction",
retry_count
);
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms))
.await;
} else {
log::error!("Channel is closed: {:?}", e);
return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
}
}
}
}
}
}
Ok(())
}
// /// 处理背压策略
// async fn handle_backpressure(
// tx: &mut mpsc::Sender<EventPretty>,
// event_pretty: EventPretty,
// backpressure_strategy: BackpressureStrategy,
// ) -> AnyResult<()> {
// match backpressure_strategy {
// BackpressureStrategy::Block => {
// // 阻塞等待，直到有空间
// if let Err(e) = tx.send(event_pretty).await {
// log::error!("Failed to send transaction to channel: {:?}", e);
// return Err(anyhow::anyhow!("Channel send failed: {:?}", e));
// }
// }
// BackpressureStrategy::Drop => {
// // 尝试发送，如果失败则丢弃
// if let Err(e) = tx.try_send(event_pretty) {
// if e.is_full() {
// log::warn!("Channel is full, dropping transaction");
// } else {
// log::error!("Channel is closed: {:?}", e);
// return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
// }
// }
// }
// BackpressureStrategy::Retry { max_attempts, wait_ms } => {
// // 重试有限次数
// let mut retry_count = 0;
// loop {
// match tx.try_send(event_pretty.clone()) {
// Ok(_) => break,
// Err(e) => {
// if e.is_full() {
// retry_count += 1;
// if retry_count >= max_attempts {
// log::warn!(
// "Channel is full after {} attempts, dropping transaction",
// retry_count
// );
// break;
// }
// tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms))
// .await;
// } else {
// log::error!("Channel is closed: {:?}", e);
// return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
// }
// }
// }
// }
// }
// }
// Ok(())
// }
}
src/streaming/grpc/types.rs
+26 -22
View File
@@ -1,5 +1,5 @@
use solana_sdk::signature::Signature;
use solana_transaction_status::{EncodedTransactionWithStatusMeta, UiTransactionEncoding};
use solana_sdk::{pubkey::Pubkey, signature::Signature};
use solana_transaction_status::TransactionWithStatusMeta;
use std::{collections::HashMap, fmt};
use yellowstone_grpc_proto::{
geyser::{
@@ -22,13 +22,14 @@ pub enum EventPretty {
#[derive(Clone)]
pub struct AccountPretty {
pub slot: u64,
pub signature: String,
pub pubkey: String,
pub signature: Signature,
pub pubkey: Pubkey,
pub executable: bool,
pub lamports: u64,
pub owner: String,
pub owner: Pubkey,
pub rent_epoch: u64,
pub data: Vec<u8>,
pub program_received_time_us: i64,
}
impl fmt::Debug for AccountPretty {
@@ -51,6 +52,7 @@ pub struct BlockMetaPretty {
pub slot: u64,
pub block_hash: String,
pub block_time: Option<Timestamp>,
pub program_received_time_us: i64,
}
impl fmt::Debug for BlockMetaPretty {
@@ -59,6 +61,7 @@ impl fmt::Debug for BlockMetaPretty {
.field("slot", &self.slot)
.field("block_hash", &self.block_hash)
.field("block_time", &self.block_time)
.field("program_received_time_us", &self.program_received_time_us)
.finish()
}
}
@@ -70,24 +73,17 @@ pub struct TransactionPretty {
pub block_time: Option<Timestamp>,
pub signature: Signature,
pub is_vote: bool,
pub tx: EncodedTransactionWithStatusMeta,
pub tx: TransactionWithStatusMeta,
pub program_received_time_us: i64,
}
impl fmt::Debug for TransactionPretty {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
struct TxWrap<'a>(&'a EncodedTransactionWithStatusMeta);
impl<'a> fmt::Debug for TxWrap<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let serialized = serde_json::to_string(self.0).expect("failed to serialize");
fmt::Display::fmt(&serialized, f)
}
}
f.debug_struct("TransactionPretty")
.field("slot", &self.slot)
.field("signature", &self.signature)
.field("is_vote", &self.is_vote)
.field("tx", &TxWrap(&self.tx))
.field("program_received_time_us", &self.program_received_time_us)
.finish()
}
}
@@ -97,13 +93,17 @@ impl From<SubscribeUpdateAccount> for AccountPretty {
let account_info = account.account.unwrap();
Self {
slot: account.slot,
signature: bs58::encode(&account_info.txn_signature.unwrap_or_default()).into_string(),
pubkey: bs58::encode(&account_info.pubkey).into_string(),
signature: Signature::try_from(
account_info.txn_signature.unwrap_or_default().as_slice(),
)
.expect("valid signature"),
pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"),
executable: account_info.executable,
lamports: account_info.lamports,
owner: bs58::encode(&account_info.owner).into_string(),
owner: Pubkey::try_from(account_info.owner.as_slice()).expect("valid pubkey"),
rent_epoch: account_info.rent_epoch,
data: account_info.data,
program_received_time_us: chrono::Utc::now().timestamp_micros(),
}
}
}
@@ -115,7 +115,12 @@ impl From<(SubscribeUpdateBlockMeta, Option<Timestamp>)> for BlockMetaPretty {
Option<Timestamp>,
),
) -> Self {
Self { block_hash: blockhash.to_string(), block_time, slot }
Self {
block_hash: blockhash.to_string(),
block_time,
slot,
program_received_time_us: chrono::Utc::now().timestamp_micros(),
}
}
}
@@ -134,9 +139,8 @@ impl From<(SubscribeUpdateTransaction, Option<Timestamp>)> for TransactionPretty
signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"),
is_vote: tx.is_vote,
tx: yellowstone_grpc_proto::convert_from::create_tx_with_meta(tx)
.expect("valid tx with meta")
.encode(UiTransactionEncoding::Base64, Some(u8::MAX), true)
.expect("failed to encode"),
.expect("valid tx with meta"),
program_received_time_us: chrono::Utc::now().timestamp_micros(),
}
}
}