gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-20 12:28:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: Major event processing system refactor for improved performance

Browse Source
This commit is contained in:
ysq
2025-08-27 21:31:31 +08:00
parent e673b0aab8
commit 9ea4dab4df
22 changed files with 829 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/grpc/stream_handler.rs
+23 -64
View File
@@ -8,7 +8,6 @@ use yellowstone_grpc_proto::geyser::{
use super::types::{BlockMetaPretty, EventPretty, TransactionPretty};
use crate::common::AnyResult;
use crate::streaming::common::EventProcessor;
use crate::streaming::event_parser::UnifiedEvent;
use crate::streaming::grpc::AccountPretty;
/// 流消息处理器
@@ -16,16 +15,12 @@ pub struct StreamHandler;
impl StreamHandler {
/// 处理单个流消息
pub async fn handle_stream_message<F>(
pub async fn handle_stream_message(
msg: SubscribeUpdate,
subscribe_tx: &mut (impl Sink<SubscribeRequest, Error = mpsc::SendError> + Unpin),
event_processor: EventProcessor,
callback: &F,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
) -> AnyResult<()> {
let created_at = msg.created_at;
match msg.update_oneof {
Some(UpdateOneof::Account(account)) => {
@@ -34,7 +29,6 @@ impl StreamHandler {
event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::Account(account_pretty),
callback,
bot_wallet,
)
.await?;
@@ -45,7 +39,6 @@ impl StreamHandler {
event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::BlockMeta(block_meta_pretty),
callback,
bot_wallet,
)
.await?;
@@ -60,7 +53,6 @@ impl StreamHandler {
event_processor
.process_grpc_event_transaction_with_metrics(
EventPretty::Transaction(transaction_pretty),
callback,
bot_wallet,
)
.await?;
@@ -84,58 +76,25 @@ impl StreamHandler {
Ok(())
}
// /// 处理背压策略
// async fn handle_backpressure(
// tx: &mut mpsc::Sender<EventPretty>,
// event_pretty: EventPretty,
// backpressure_strategy: BackpressureStrategy,
// ) -> AnyResult<()> {
// match backpressure_strategy {
// BackpressureStrategy::Block => {
// // 阻塞等待，直到有空间
// if let Err(e) = tx.send(event_pretty).await {
// log::error!("Failed to send transaction to channel: {:?}", e);
// return Err(anyhow::anyhow!("Channel send failed: {:?}", e));
// }
// }
// BackpressureStrategy::Drop => {
// // 尝试发送，如果失败则丢弃
// if let Err(e) = tx.try_send(event_pretty) {
// if e.is_full() {
// log::warn!("Channel is full, dropping transaction");
// } else {
// log::error!("Channel is closed: {:?}", e);
// return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
// }
// }
// }
// BackpressureStrategy::Retry { max_attempts, wait_ms } => {
// // 重试有限次数
// let mut retry_count = 0;
// loop {
// match tx.try_send(event_pretty.clone()) {
// Ok(_) => break,
// Err(e) => {
// if e.is_full() {
// retry_count += 1;
// if retry_count >= max_attempts {
// log::warn!(
// "Channel is full after {} attempts, dropping transaction",
// retry_count
// );
// break;
// }
// tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms))
// .await;
// } else {
// log::error!("Channel is closed: {:?}", e);
// return Err(anyhow::anyhow!("Channel is closed: {:?}", e));
// }
// }
// }
// }
// }
// }
// Ok(())
// }
pub async fn handle_stream_system_message(
msg: SubscribeUpdate,
subscribe_tx: &mut (impl Sink<SubscribeRequest, Error = mpsc::SendError> + Unpin),
) -> AnyResult<Option<EventPretty>> {
let created_at = msg.created_at;
let event_pretty = match msg.update_oneof {
Some(UpdateOneof::Transaction(sut)) => Some(TransactionPretty::from((sut, created_at))),
Some(UpdateOneof::Ping(_)) => {
subscribe_tx
.send(SubscribeRequest {
ping: Some(SubscribeRequestPing { id: 1 }),
..Default::default()
})
.await?;
None
}
Some(UpdateOneof::Pong(_)) => None,
_ => None,
};
Ok(event_pretty.map(|e| EventPretty::Transaction(e)))
}
}
src/streaming/grpc/types.rs
+7 -6
View File
@@ -69,7 +69,7 @@ impl fmt::Debug for BlockMetaPretty {
#[derive(Clone)]
pub struct TransactionPretty {
pub slot: u64,
pub transaction_index: Option<u64>, // 新增：交易在slot中的索引
pub transaction_index: Option<u64>, // 新增：交易在slot中的索引
pub block_hash: String,
pub block_time: Option<Timestamp>,
pub signature: Signature,
@@ -95,10 +95,11 @@ impl From<SubscribeUpdateAccount> for AccountPretty {
let account_info = account.account.unwrap();
Self {
slot: account.slot,
signature: Signature::try_from(
account_info.txn_signature.unwrap_or_default().as_slice(),
)
.expect("valid signature"),
signature: if let Some(txn_signature) = account_info.txn_signature {
Signature::try_from(txn_signature.as_slice()).expect("valid signature")
} else {
Signature::default()
},
pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"),
executable: account_info.executable,
lamports: account_info.lamports,
@@ -138,7 +139,7 @@ impl From<(SubscribeUpdateTransaction, Option<Timestamp>)> for TransactionPretty
let transaction_index = tx.index;
Self {
slot,
transaction_index: Some(transaction_index), // 提取交易索引
transaction_index: Some(transaction_index), // 提取交易索引
block_time,
block_hash: "".to_string(),
signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"),