gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-17 19:08:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enhance event filtering and monitoring system

Browse Source 
- Add EventTypeFilter for precise event type filtering
- Refactor metrics to track events by type (tx/account/block)
- Add parser caching for improved performance
- Enhance gRPC subscription management
- Update examples and type definitions
This commit is contained in:
ysq
2025-08-16 14:48:51 +08:00
parent 89f4a85a11
commit ea7ecd1d0c
15 changed files with 477 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/grpc/event_processor.rs
+54 -20
View File
@@ -1,12 +1,14 @@
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use solana_sdk::pubkey::Pubkey;
use super::types::EventPretty;
use crate::common::AnyResult;
use crate::streaming::common::{
EventBatchProcessor as EventBatchCollector, MetricsManager, StreamClientConfig as ClientConfig,
EventBatchProcessor as EventBatchCollector, MetricsEventType, MetricsManager,
StreamClientConfig as ClientConfig,
};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
use crate::streaming::event_parser::EventParser;
@@ -18,12 +20,29 @@ use crate::streaming::event_parser::{
pub struct EventProcessor {
pub(crate) metrics_manager: MetricsManager,
pub(crate) config: ClientConfig,
pub(crate) parser_cache: Arc<Mutex<Option<Arc<dyn EventParser>>>>,
}
impl EventProcessor {
/// 创建新的事件处理器
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
Self { metrics_manager, config }
Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) }
}
/// 获取或创建解析器，使用缓存机制避免重复创建
fn get_or_create_parser(
&self,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> Arc<dyn EventParser> {
let mut cache = self.parser_cache.lock().unwrap();
if let Some(cached_parser) = cache.clone() {
return cached_parser.clone();
}
let parser: Arc<dyn EventParser> =
Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone()));
*cache = Some(parser.clone());
parser
}
/// 使用性能监控处理事件交易
@@ -33,19 +52,21 @@ impl EventProcessor {
callback: &F,
bot_wallet: Option<Pubkey>,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_process_count().await;
self.metrics_manager.add_account_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let account_event = AccountEventParser::parse_account_event(
protocols.clone(),
account_pretty,
program_received_time_ms,
event_type_filter,
);
if let Some(event) = account_event {
callback(event);
@@ -53,21 +74,22 @@ impl EventProcessor {
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(1, processing_time_ms).await;
self.metrics_manager
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_process_count().await;
self.metrics_manager.add_tx_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature.to_string();
// 直接创建解析器并处理事务
let parser: Arc<dyn EventParser> =
Arc::new(MutilEventParser::new(protocols.clone()));
// 使用缓存获取解析器
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
let all_events = parser
.parse_transaction(
transaction_pretty.tx.clone(),
@@ -98,13 +120,15 @@ impl EventProcessor {
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(event_count as u64, processing_time_ms).await;
self.metrics_manager
.update_metrics(MetricsEventType::Tx, event_count as u64, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, event_count);
}
EventPretty::BlockMeta(block_meta_pretty) => {
let start_time = std::time::Instant::now();
self.metrics_manager.add_process_count().await;
self.metrics_manager.add_block_meta_process_count().await;
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
@@ -119,7 +143,9 @@ impl EventProcessor {
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(1, processing_time_ms).await;
self.metrics_manager
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
@@ -135,19 +161,21 @@ impl EventProcessor {
batch_processor: &mut EventBatchCollector<F>,
bot_wallet: Option<Pubkey>,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<()>
where
F: Fn(Vec<Box<dyn UnifiedEvent>>) + Send + Sync + 'static,
{
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_process_count().await;
self.metrics_manager.add_account_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let account_event = AccountEventParser::parse_account_event(
protocols.clone(),
account_pretty,
program_received_time_ms,
event_type_filter,
);
if let Some(event) = account_event {
(batch_processor.callback)(vec![event]);
@@ -155,21 +183,22 @@ impl EventProcessor {
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.metrics_manager.update_metrics(1, processing_time_ms).await;
self.metrics_manager
.update_metrics(MetricsEventType::Account, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_process_count().await;
self.metrics_manager.add_tx_process_count().await;
let start_time = std::time::Instant::now();
let program_received_time_ms = chrono::Utc::now().timestamp_millis();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature.to_string();
// 直接创建解析器并处理事务
let parser: Arc<dyn EventParser> =
Arc::new(MutilEventParser::new(protocols.clone()));
// 使用缓存获取解析器
let parser = self.get_or_create_parser(protocols.clone(), event_type_filter);
let result = parser
.parse_transaction(
transaction_pretty.tx.clone(),
@@ -224,13 +253,16 @@ impl EventProcessor {
let processing_time_ms = processing_time.as_millis() as f64;
// 实际调用性能指标更新
self.metrics_manager.update_metrics(total_events as u64, processing_time_ms).await;
self.metrics_manager
.update_metrics(MetricsEventType::Tx, total_events as u64, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, total_events);
}
EventPretty::BlockMeta(block_meta_pretty) => {
let start_time = std::time::Instant::now();
self.metrics_manager.add_block_meta_process_count().await;
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
@@ -245,7 +277,9 @@ impl EventProcessor {
let processing_time = start_time.elapsed();
let processing_time_ms = processing_time.as_millis() as f64;
// 更新性能指标（如果启用）
self.metrics_manager.update_metrics(1, processing_time_ms).await;
self.metrics_manager
.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms)
.await;
// 记录慢处理操作
self.metrics_manager.log_slow_processing(processing_time_ms, 1);
}
src/streaming/grpc/subscription.rs
+28 -5
View File
@@ -12,6 +12,7 @@ use super::types::AccountsFilterMap;
use super::types::TransactionsFilterMap;
use crate::common::AnyResult;
use crate::streaming::common::StreamClientConfig as ClientConfig;
use crate::streaming::event_parser::common::filter::EventTypeFilter;
/// 订阅管理器
#[derive(Clone)]
@@ -41,17 +42,27 @@ impl SubscriptionManager {
/// 创建订阅请求并返回流
pub async fn subscribe_with_request(
&self,
transactions: TransactionsFilterMap,
transactions: Option<TransactionsFilterMap>,
accounts: Option<AccountsFilterMap>,
commitment: Option<CommitmentLevel>,
event_type_filter: Option<EventTypeFilter>,
) -> AnyResult<(
impl Sink<SubscribeRequest, Error = mpsc::SendError>,
impl Stream<Item = Result<SubscribeUpdate, Status>>,
)> {
let blocks_meta = if event_type_filter.is_some()
&& event_type_filter.as_ref().unwrap().include_block_event()
{
hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} }
} else if event_type_filter.is_none() {
hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} }
} else {
hashmap! {}
};
let subscribe_request = SubscribeRequest {
accounts: accounts.unwrap_or_default(),
transactions,
blocks_meta: hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} },
transactions: transactions.unwrap_or_default(),
blocks_meta,
commitment: if let Some(commitment) = commitment {
Some(commitment as i32)
} else {
@@ -69,10 +80,16 @@ impl SubscriptionManager {
&self,
account: Vec<String>,
owner: Vec<String>,
event_type_filter: Option<EventTypeFilter>,
) -> Option<AccountsFilterMap> {
if account.len() == 0 && owner.len() == 0 {
return None;
}
if event_type_filter.is_some()
&& !event_type_filter.as_ref().unwrap().include_account_event()
{
return None;
}
let mut accounts = HashMap::new();
accounts.insert(
"".to_owned(),
@@ -92,7 +109,13 @@ impl SubscriptionManager {
account_include: Vec<String>,
account_exclude: Vec<String>,
account_required: Vec<String>,
) -> TransactionsFilterMap {
event_type_filter: Option<EventTypeFilter>,
) -> Option<TransactionsFilterMap> {
if event_type_filter.is_some()
&& !event_type_filter.as_ref().unwrap().include_transaction_event()
{
return None;
}
let mut transactions = HashMap::new();
transactions.insert(
"client".to_string(),
@@ -105,7 +128,7 @@ impl SubscriptionManager {
account_required,
},
);
transactions
Some(transactions)
}
/// 获取配置