From c3d599acd52a2362aa405f57bd83637b6fd98468 Mon Sep 17 00:00:00 2001 From: vnxfsc <1047658287@qq.com> Date: Mon, 25 Aug 2025 00:37:31 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E4=BA=A4?= =?UTF-8?q?=E6=98=93=E7=B4=A2=E5=BC=95=E5=8A=9F=E8=83=BD=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 为 TransactionPretty 添加 transaction_index 字段 - 从 Yellowstone gRPC 的 SubscribeUpdateTransaction.transaction.index 提取交易索引 - 为 EventMetadata 添加 transaction_index 字段用于存储交易在 slot 中的索引 - 为 UnifiedEvent trait 添加 transaction_index() 和 set_transaction_index() 方法 - 更新事件处理器以传递交易索引到事件元数据 - 更新主程序日志以显示交易索引信息 交易事件现在包含它们在 slot 中的索引位置，账户事件由于 gRPC 协议限制保持为 None --- src/main.rs | 9 ++++++++- src/streaming/event_parser/common/mod.rs | 8 ++++++++ src/streaming/event_parser/common/types.rs | 9 ++++++++- src/streaming/event_parser/core/traits.rs | 6 ++++++ src/streaming/grpc/event_processor.rs | 7 ++++++- src/streaming/grpc/types.rs | 5 +++++ 6 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index dd110cd..6208408 100755 --- a/src/main.rs +++ b/src/main.rs @@ -188,7 +188,14 @@ async fn test_shreds() -> Result<(), Box> { fn create_event_callback() -> impl Fn(Box) { |event: Box| { - println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id()); + println!( + "🎉 Event received! Type: {:?}, ID: {}, Slot: {}, Transaction Index: {:?}, Instruction Index: {}", + event.event_type(), + event.id(), + event.slot(), + event.transaction_index(), + event.index() + ); match_event!(event, { // -------------------------- block meta ----------------------- BlockMetaEvent => |e: BlockMetaEvent| { diff --git a/src/streaming/event_parser/common/mod.rs b/src/streaming/event_parser/common/mod.rs index 3ecc161..bc4f9d0 100755 --- a/src/streaming/event_parser/common/mod.rs +++ b/src/streaming/event_parser/common/mod.rs @@ -63,6 +63,14 @@ macro_rules! impl_unified_event { fn index(&self) -> String { self.metadata.index.clone() } + + fn transaction_index(&self) -> Option { + self.metadata.transaction_index + } + + fn set_transaction_index(&mut self, transaction_index: Option) { + self.metadata.set_transaction_index(transaction_index); + } } }; } diff --git a/src/streaming/event_parser/common/types.rs b/src/streaming/event_parser/common/types.rs index 9794cc8..4eaa99f 100755 --- a/src/streaming/event_parser/common/types.rs +++ b/src/streaming/event_parser/common/types.rs @@ -343,6 +343,7 @@ pub struct EventMetadata { pub id: String, pub signature: String, pub slot: u64, + pub transaction_index: Option, // 新增：交易在slot中的索引 pub block_time: i64, pub block_time_ms: i64, pub program_received_time_ms: i64, @@ -352,7 +353,7 @@ pub struct EventMetadata { pub program_id: Pubkey, pub transfer_datas: Vec, pub swap_data: Option, - pub index: String, + pub index: String, // 保留原有的指令索引 } impl EventMetadata { @@ -373,6 +374,7 @@ impl EventMetadata { id, signature, slot, + transaction_index: None, // 默认为None，后续设置 block_time, block_time_ms, program_received_time_ms, @@ -403,6 +405,11 @@ impl EventMetadata { self.swap_data = swap_data; } + /// 设置交易索引 + pub fn set_transaction_index(&mut self, transaction_index: Option) { + self.transaction_index = transaction_index; + } + /// Recycle EventMetadata to object pool pub async fn recycle(self) { EVENT_METADATA_POOL.release(self).await; diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index a6d8574..a23d53c 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -67,6 +67,12 @@ pub trait UnifiedEvent: Debug + Send + Sync { /// Get index fn index(&self) -> String; + + /// Get transaction index in slot + fn transaction_index(&self) -> Option; + + /// Set transaction index in slot + fn set_transaction_index(&mut self, transaction_index: Option); } /// 事件解析器trait - 定义了事件解析的核心方法 diff --git a/src/streaming/grpc/event_processor.rs b/src/streaming/grpc/event_processor.rs index 10454fe..18ebfdb 100644 --- a/src/streaming/grpc/event_processor.rs +++ b/src/streaming/grpc/event_processor.rs @@ -90,7 +90,7 @@ impl EventProcessor { // 使用缓存获取解析器 let parser = self.get_or_create_parser(protocols.clone(), event_type_filter); - let all_events = parser + let mut all_events = parser .parse_transaction( transaction_pretty.tx.clone(), &signature, @@ -105,6 +105,11 @@ impl EventProcessor { .await .unwrap_or_else(|_e| vec![]); + // 为所有事件设置交易索引 + for event in &mut all_events { + event.set_transaction_index(transaction_pretty.transaction_index); + } + // 保存事件数量用于日志记录 let event_count = all_events.len(); diff --git a/src/streaming/grpc/types.rs b/src/streaming/grpc/types.rs index 8d80a7c..d75b986 100644 --- a/src/streaming/grpc/types.rs +++ b/src/streaming/grpc/types.rs @@ -66,6 +66,7 @@ impl fmt::Debug for BlockMetaPretty { #[derive(Clone)] pub struct TransactionPretty { pub slot: u64, + pub transaction_index: Option, // 新增：交易在slot中的索引 pub block_hash: String, pub block_time: Option, pub signature: Signature, @@ -85,6 +86,7 @@ impl fmt::Debug for TransactionPretty { f.debug_struct("TransactionPretty") .field("slot", &self.slot) + .field("transaction_index", &self.transaction_index) .field("signature", &self.signature) .field("is_vote", &self.is_vote) .field("tx", &TxWrap(&self.tx)) @@ -127,8 +129,11 @@ impl From<(SubscribeUpdateTransaction, Option)> for TransactionPretty ), ) -> Self { let tx = transaction.expect("should be defined"); + // 根据用户说明，交易索引在 transaction.index 中 + let transaction_index = tx.index; Self { slot, + transaction_index: Some(transaction_index), // 提取交易索引 block_time, block_hash: "".to_string(), signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"), From dee683396d3493cddb89383486a41d76521123a2 Mon Sep 17 00:00:00 2001 From: ysq Date: Tue, 26 Aug 2025 17:57:39 +0800 Subject: [PATCH 2/7] perf: Refactor event processing system for better performance --- Cargo.toml | 3 +- ...se_tx_events.rs => parse_tx_events.rs.bak} | 83 ++- src/streaming/common/constants.rs | 2 +- src/streaming/common/event_processor.rs | 254 +++++++++ src/streaming/common/metrics.rs | 173 +++--- src/streaming/common/mod.rs | 2 + src/streaming/common/subscription.rs | 12 +- src/streaming/event_parser/common/mod.rs | 22 +- src/streaming/event_parser/common/types.rs | 517 ++++++++---------- src/streaming/event_parser/common/utils.rs | 12 - .../event_parser/core/account_event_parser.rs | 11 +- .../event_parser/core/common_event_parser.rs | 11 +- src/streaming/event_parser/core/traits.rs | 329 +++++------ .../protocols/block/block_meta_event.rs | 9 +- .../event_parser/protocols/bonk/parser.rs | 22 +- .../event_parser/protocols/mutil/parser.rs | 52 +- .../event_parser/protocols/pumpfun/parser.rs | 17 +- .../event_parser/protocols/pumpswap/parser.rs | 17 +- .../protocols/raydium_amm_v4/parser.rs | 17 +- .../protocols/raydium_clmm/parser.rs | 17 +- .../protocols/raydium_cpmm/parser.rs | 17 +- src/streaming/grpc/event_processor.rs | 290 ---------- src/streaming/grpc/mod.rs | 22 +- src/streaming/grpc/stream_handler.rs | 165 +++--- src/streaming/grpc/types.rs | 48 +- src/streaming/mod.rs | 4 +- src/streaming/shred/connection.rs | 15 +- src/streaming/shred/event_processor.rs | 170 ------ src/streaming/shred/mod.rs | 8 +- src/streaming/shred/stream_handler.rs | 116 ---- src/streaming/shred_stream.rs | 167 ++---- src/streaming/yellowstone_grpc.rs | 205 +------ src/streaming/yellowstone_sub_system.rs | 15 +- 33 files changed, 1158 insertions(+), 1666 deletions(-) rename examples/{parse_tx_events.rs => parse_tx_events.rs.bak} (53%) create mode 100644 src/streaming/common/event_processor.rs delete mode 100644 src/streaming/grpc/event_processor.rs delete mode 100644 src/streaming/shred/event_processor.rs delete mode 100644 src/streaming/shred/stream_handler.rs diff --git a/Cargo.toml b/Cargo.toml index 3147f66..040d079 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,6 @@ serde-big-array = "0.5.1" futures = "0.3.31" futures-util = "0.3.31" base64 = "0.22.1" -bs58 = "0.5.1" rand = "0.9.0" bincode = "1.3.3" anyhow = "1.0.90" @@ -64,3 +63,5 @@ borsh-derive = "1.5.5" indicatif = "0.18.0" maplit = "1.0.2" env_logger = "0.11.8" +crossbeam = "0.8.4" +crossbeam-queue = "0.3.12" diff --git a/examples/parse_tx_events.rs b/examples/parse_tx_events.rs.bak similarity index 53% rename from examples/parse_tx_events.rs rename to examples/parse_tx_events.rs.bak index cbb5bb6..4ce1b6b 100644 --- a/examples/parse_tx_events.rs +++ b/examples/parse_tx_events.rs.bak @@ -1,8 +1,13 @@ use anyhow::Result; use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::message::v0::LoadedAddresses; use solana_streamer_sdk::streaming::event_parser::{ protocols::MutilEventParser, EventParser, Protocol, }; +use solana_transaction_status::{ + option_serializer::OptionSerializer, TransactionStatusMeta, TransactionWithStatusMeta, + VersionedTransactionWithStatusMeta, +}; use std::str::FromStr; use std::sync::Arc; @@ -10,7 +15,7 @@ use std::sync::Arc; #[tokio::main] async fn main() -> Result<()> { let signatures = vec![ - "42agNk1heHabNAVRzEKqQEt5adGkQzRYf9M1Q81uBJPCCHyP4cyCA1RNkgxXrtEAWeeGcytyh2TsnkBDgqnHeq4z", + "5cnxDiHzUTUutMwnTCsvnMhyL9jEQsRWimmWU1gKxpQBngDaGTkou1YbGhUJhAhmTgvu49PYMmFQbbR38wdZDxJF", ]; // Validate signature format let mut valid_signatures = Vec::new(); @@ -52,7 +57,7 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> { .get_transaction_with_config( &signature, solana_client::rpc_config::RpcTransactionConfig { - encoding: Some(UiTransactionEncoding::Binary), + encoding: Some(UiTransactionEncoding::Base64), commitment: Some(CommitmentConfig::confirmed()), max_supported_transaction_version: Some(0), }, @@ -99,21 +104,85 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> { ]; let parser: Arc = Arc::new(MutilEventParser::new(protocols, None)); let start_time = std::time::Instant::now(); + + // 从 EncodedTransaction 获取 VersionedTransaction + let versioned_tx = match transaction.transaction.transaction.decode() { + Some(tx) => tx, + None => { + println!("Failed to decode transaction"); + return Ok(()); + } + }; + + // 创建 TransactionWithStatusMeta + let tx = TransactionWithStatusMeta::Complete(VersionedTransactionWithStatusMeta { + transaction: versioned_tx, + meta: TransactionStatusMeta { + status: Ok(()), + fee: transaction.transaction.meta.as_ref().map_or(0, |m| m.fee), + pre_balances: transaction + .transaction + .meta + .as_ref() + .map_or(vec![], |m| m.pre_balances.clone()), + post_balances: transaction + .transaction + .meta + .as_ref() + .map_or(vec![], |m| m.post_balances.clone()), + inner_instructions: transaction.transaction.meta.as_ref().and_then(|m| { + if let OptionSerializer::Some(inner_instructions) = &m.inner_instructions { + // 手动将每个UiInnerInstructions转换为InnerInstructions + Some(inner_instructions.iter().map(|ui_inner| { + solana_transaction_status::InnerInstructions { + index: ui_inner.index, + instructions: ui_inner.instructions.iter().map(|ui_inst| { + solana_transaction_status::InnerInstruction { + instruction: solana_sdk::instruction::Instruction { + program_id: solana_sdk::pubkey::Pubkey::new_from_array([0; 32]), + accounts: vec![], + data: vec![], + }, + stack_height: None, + } + }).collect(), + } + }).collect()) + } else { + None + } + }), + log_messages: transaction.transaction.meta.as_ref().and_then(|m| { + if let OptionSerializer::Some(logs) = &m.log_messages { + Some(logs.clone()) + } else { + None + } + }), + pre_token_balances: None, + post_token_balances: None, + rewards: None, + loaded_addresses: LoadedAddresses::default(), + return_data: None, + compute_units_consumed: None, + cost_units: None, + }, + }); + + // TransactionWithStatusMeta let events = parser .parse_transaction( - transaction.transaction.clone(), + tx, &signature.to_string(), Some(transaction.slot), None, - 0, + chrono::Utc::now().timestamp_micros(), None, ) .await .unwrap_or_else(|_e| vec![]); - let end_time = std::time::Instant::now(); - let duration = end_time.duration_since(start_time); - println!("Parsing time: {:?}", duration); + println!("Parsing time: {:?}", start_time.elapsed()); for event in events { println!("{:?}\n", event); } diff --git a/src/streaming/common/constants.rs b/src/streaming/common/constants.rs index d4b3d18..8cc2be8 100644 --- a/src/streaming/common/constants.rs +++ b/src/streaming/common/constants.rs @@ -11,4 +11,4 @@ pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5; // 性能监控相关常量 pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5; pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10; -pub const SLOW_PROCESSING_THRESHOLD_MS: f64 = 10.0; +pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 500.0; diff --git a/src/streaming/common/event_processor.rs b/src/streaming/common/event_processor.rs new file mode 100644 index 0000000..e69d699 --- /dev/null +++ b/src/streaming/common/event_processor.rs @@ -0,0 +1,254 @@ +use std::sync::Arc; +use tokio::task::JoinHandle; + +use solana_sdk::pubkey::Pubkey; + +use crate::common::AnyResult; +use crate::streaming::common::{ + MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig, +}; +use crate::streaming::event_parser::common::filter::EventTypeFilter; +use crate::streaming::event_parser::core::account_event_parser::AccountEventParser; +use crate::streaming::event_parser::core::common_event_parser::CommonEventParser; +use crate::streaming::event_parser::EventParser; +use crate::streaming::event_parser::{ + core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, +}; +use crate::streaming::grpc::{BackpressureStrategy, BatchConfig, EventPretty}; +use crate::streaming::shred::TransactionWithSlot; +use once_cell::sync::OnceCell; + +/// 事件处理器 +pub struct EventProcessor { + pub(crate) metrics_manager: MetricsManager, + pub(crate) config: ClientConfig, + pub(crate) parser_cache: OnceCell>, + pub(crate) protocols: Vec, + pub(crate) event_type_filter: Option, + pub(crate) backpressure_strategy: BackpressureStrategy, + pub(crate) batch_config: BatchConfig, +} + +impl EventProcessor { + /// 创建新的事件处理器 + pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { + Self { + metrics_manager, + config, + parser_cache: OnceCell::new(), + protocols: vec![], + event_type_filter: None, + backpressure_strategy: BackpressureStrategy::Block, + batch_config: BatchConfig::default(), + } + } + + pub fn set_protocols_and_event_type_filter( + &mut self, + protocols: Vec, + event_type_filter: Option, + backpressure_strategy: BackpressureStrategy, + batch_config: BatchConfig, + ) { + self.protocols = protocols.clone(); + self.event_type_filter = event_type_filter.clone(); + self.backpressure_strategy = backpressure_strategy; + self.batch_config = batch_config; + self.parser_cache + .get_or_init(|| Arc::new(MutilEventParser::new(protocols, event_type_filter))); + } + + pub fn get_parser(&self) -> Arc { + self.parser_cache.get().unwrap().clone() + } + + pub fn get_event_handle(&self) -> Option> { + return None; + } + + pub async fn process_grpc_event_transaction_with_metrics( + &self, + event_pretty: EventPretty, + callback: &F, + bot_wallet: Option, + ) -> AnyResult<()> + where + F: Fn(Box) + Send + Sync, + { + self.process_grpc_event_transaction(event_pretty, callback, bot_wallet).await?; + Ok(()) + } + + async fn process_grpc_event_transaction( + &self, + event_pretty: EventPretty, + callback: &F, + bot_wallet: Option, + ) -> AnyResult<()> + where + F: Fn(Box) + Send + Sync, + { + match event_pretty { + EventPretty::Account(account_pretty) => { + self.metrics_manager.add_account_process_count(); + let account_event = AccountEventParser::parse_account_event( + self.protocols.clone(), + account_pretty, + self.event_type_filter.clone(), + ); + if let Some(event) = account_event { + let processing_time_us = event.program_handle_time_consuming_us() as f64; + callback(event); + // 更新性能指标（如果启用） + self.metrics_manager.update_metrics( + MetricsEventType::Account, + 1, + processing_time_us, + ); + } + } + EventPretty::Transaction(transaction_pretty) => { + self.metrics_manager.add_tx_process_count(); + let slot = transaction_pretty.slot; + let signature = transaction_pretty.signature; + // 使用缓存获取解析器 + let parser = self.get_parser(); + let all_events = parser + .parse_transaction( + transaction_pretty.tx.clone(), + signature, + Some(slot), + transaction_pretty.block_time, + transaction_pretty.program_received_time_us, + bot_wallet, + ) + .await + .unwrap_or_else(|_e| vec![]); + + let max_time_consuming_us = all_events + .iter() + .map(|event| event.program_handle_time_consuming_us()) + .max() + .unwrap_or(0); + + // 保存事件数量用于日志记录 + let event_count = all_events.len(); + + // 批量处理事件 + if !all_events.is_empty() { + for mut event in all_events { + event.set_program_handle_time_consuming_us( + chrono::Utc::now().timestamp_micros() + - event.program_received_time_us(), + ); + callback(event); + } + } + + // 更新性能指标 + // 更新性能指标（如果启用） + self.metrics_manager.update_metrics( + MetricsEventType::Tx, + event_count as u64, + max_time_consuming_us as f64, + ); + } + EventPretty::BlockMeta(block_meta_pretty) => { + self.metrics_manager.add_block_meta_process_count(); + let block_time_ms = block_meta_pretty + .block_time + .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000) + .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); + let block_meta_event = CommonEventParser::generate_block_meta_event( + block_meta_pretty.slot, + &block_meta_pretty.block_hash, + block_time_ms, + block_meta_pretty.program_received_time_us, + ); + let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64; + callback(block_meta_event); + // 更新性能指标（如果启用） + self.metrics_manager.update_metrics( + MetricsEventType::BlockMeta, + 1, + processing_time_us, + ); + } + } + + Ok(()) + } + + /// 即时处理单个交易 + pub async fn process_shred_transaction_immediate( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + callback: &F, + ) -> AnyResult<()> + where + F: Fn(Box) + Send + Sync, + { + self.metrics_manager.add_tx_process_count(); + let program_received_time_us = chrono::Utc::now().timestamp_micros(); + let slot = transaction_with_slot.slot; + let versioned_tx = transaction_with_slot.transaction; + let signature = versioned_tx.signatures[0]; + + // 获取缓存的解析器 + let parser = self.get_parser(); + + let all_events = parser + .parse_versioned_transaction( + &versioned_tx, + signature, + Some(slot), + None, + program_received_time_us, + bot_wallet, + ) + .await + .unwrap_or_else(|_e| vec![]); + + let max_time_consuming_us = all_events + .iter() + .map(|event| event.program_handle_time_consuming_us()) + .max() + .unwrap_or(0); + + // 保存事件数量用于日志记录 + let event_count = all_events.len(); + + // 即时处理事件 + for mut event in all_events { + event.set_program_handle_time_consuming_us( + chrono::Utc::now().timestamp_micros() - event.program_received_time_us(), + ); + callback(event); + } + + // 实际调用性能指标更新 + self.metrics_manager.update_metrics( + MetricsEventType::Tx, + event_count as u64, + max_time_consuming_us as f64, + ); + + Ok(()) + } +} + +// 实现 Clone trait 以支持模块间共享 +impl Clone for EventProcessor { + fn clone(&self) -> Self { + Self { + metrics_manager: self.metrics_manager.clone(), + config: self.config.clone(), + parser_cache: self.parser_cache.clone(), + protocols: self.protocols.clone(), + event_type_filter: self.event_type_filter.clone(), + backpressure_strategy: self.backpressure_strategy.clone(), + batch_config: self.batch_config.clone(), + } + } +} diff --git a/src/streaming/common/metrics.rs b/src/streaming/common/metrics.rs index 2ccdebe..6dd80e5 100644 --- a/src/streaming/common/metrics.rs +++ b/src/streaming/common/metrics.rs @@ -1,5 +1,7 @@ use std::sync::Arc; -use tokio::sync::Mutex; +use crossbeam::utils::Backoff; +use crossbeam::atomic::AtomicCell; +use std::sync::RwLock; use super::config::StreamClientConfig; use super::constants::*; @@ -31,9 +33,9 @@ impl EventMetrics { pub struct PerformanceMetrics { pub start_time: std::time::Instant, pub event_metrics: [EventMetrics; 3], // [Tx, Account, BlockMeta] - pub average_processing_time_ms: f64, - pub min_processing_time_ms: f64, - pub max_processing_time_ms: f64, + pub average_processing_time_us: f64, + pub min_processing_time_us: f64, + pub max_processing_time_us: f64, pub last_update_time: std::time::Instant, } @@ -65,9 +67,9 @@ impl PerformanceMetrics { Self { start_time: now, event_metrics: [EventMetrics::new(now), EventMetrics::new(now), EventMetrics::new(now)], - average_processing_time_ms: 0.0, - min_processing_time_ms: 0.0, - max_processing_time_ms: 0.0, + average_processing_time_us: 0.0, + min_processing_time_us: 0.0, + max_processing_time_us: 0.0, last_update_time: now, } } @@ -132,7 +134,7 @@ impl PerformanceMetrics { /// 通用性能监控管理器 pub struct MetricsManager { - metrics: Arc>, + metrics: Arc>, config: Arc, stream_name: String, } @@ -140,7 +142,7 @@ pub struct MetricsManager { impl MetricsManager { /// 创建新的性能监控管理器 pub fn new( - metrics: Arc>, + metrics: Arc>, config: Arc, stream_name: String, ) -> Self { @@ -148,14 +150,24 @@ impl MetricsManager { } /// 获取性能指标 - pub async fn get_metrics(&self) -> PerformanceMetrics { - let metrics = self.metrics.lock().await; - metrics.clone() + pub fn get_metrics(&self) -> PerformanceMetrics { + // 使用 Backoff 策略进行读取尝试 + let backoff = Backoff::new(); + loop { + match self.metrics.read() { + Ok(metrics) => return metrics.clone(), + Err(_) => { + // 如果获取读锁失败，使用指数退避策略 + backoff.snooze(); + continue; + } + } + } } /// 打印性能指标 - pub async fn print_metrics(&self) { - let metrics = self.get_metrics().await; + pub fn print_metrics(&self) { + let metrics = self.get_metrics(); let event_names = ["TX", "Account", "Block Meta"]; let event_types = [MetricsEventType::Tx, MetricsEventType::Account, MetricsEventType::BlockMeta]; @@ -189,11 +201,11 @@ impl MetricsManager { // 打印处理时间统计表格 println!("\n⏱️ Processing Time Statistics"); println!("┌─────────────────────┬─────────────┐"); - println!("│ Metric │ Value (ms) │"); + println!("│ Metric │ Value (us) │"); println!("├─────────────────────┼─────────────┤"); - println!("│ Average │ {:9.2} │", metrics.average_processing_time_ms); - println!("│ Minimum │ {:9.2} │", metrics.min_processing_time_ms); - println!("│ Maximum │ {:9.2} │", metrics.max_processing_time_ms); + println!("│ Average │ {:9.2} │", metrics.average_processing_time_us); + println!("│ Minimum │ {:9.2} │", metrics.min_processing_time_us); + println!("│ Maximum │ {:9.2} │", metrics.max_processing_time_us); println!("└─────────────────────┴─────────────┘"); println!(); } @@ -212,91 +224,118 @@ impl MetricsManager { )); loop { interval.tick().await; - metrics_manager.print_metrics().await; + metrics_manager.print_metrics(); } }); Some(handle) } /// 更新处理次数 - pub async fn add_process_count(&self, event_type: MetricsEventType) { + pub fn add_process_count(&self, event_type: MetricsEventType) { if !self.config.enable_metrics { return; } - let mut metrics = self.metrics.lock().await; - metrics.event_metrics[event_type.as_index()].process_count += 1; + + // 使用 Backoff 策略进行写入尝试 + let backoff = Backoff::new(); + loop { + match self.metrics.write() { + Ok(mut metrics) => { + metrics.event_metrics[event_type.as_index()].process_count += 1; + break; + }, + Err(_) => { + // 如果获取写锁失败，使用指数退避策略 + backoff.snooze(); + continue; + } + } + } } // 保持向后兼容的方法 - pub async fn add_tx_process_count(&self) { - self.add_process_count(MetricsEventType::Tx).await; + pub fn add_tx_process_count(&self) { + self.add_process_count(MetricsEventType::Tx); } - pub async fn add_account_process_count(&self) { - self.add_process_count(MetricsEventType::Account).await; + pub fn add_account_process_count(&self) { + self.add_process_count(MetricsEventType::Account); } - pub async fn add_block_meta_process_count(&self) { - self.add_process_count(MetricsEventType::BlockMeta).await; + pub fn add_block_meta_process_count(&self) { + self.add_process_count(MetricsEventType::BlockMeta); } /// 更新性能指标 - pub async fn update_metrics( + pub fn update_metrics( &self, event_type: MetricsEventType, events_processed: u64, - processing_time_ms: f64, + processing_time_us: f64, ) { // 检查是否启用性能监控 if !self.config.enable_metrics { return; } + + // 使用 Backoff 策略进行写入尝试 + let backoff = Backoff::new(); + loop { + match self.metrics.write() { + Ok(mut metrics) => { + let now = std::time::Instant::now(); + let index = event_type.as_index(); - let mut metrics = self.metrics.lock().await; - let now = std::time::Instant::now(); - let index = event_type.as_index(); + // 更新事件计数 + metrics.event_metrics[index].events_processed += events_processed; + metrics.event_metrics[index].events_in_window += events_processed; - // 更新事件计数 - metrics.event_metrics[index].events_processed += events_processed; - metrics.event_metrics[index].events_in_window += events_processed; + metrics.last_update_time = now; - metrics.last_update_time = now; + // 更新处理时间统计 + if processing_time_us < metrics.min_processing_time_us + || metrics.min_processing_time_us == 0.0 + { + metrics.min_processing_time_us = processing_time_us; + } + if processing_time_us > metrics.max_processing_time_us { + metrics.max_processing_time_us = processing_time_us; + } - // 更新处理时间统计 - if processing_time_ms < metrics.min_processing_time_ms - || metrics.min_processing_time_ms == 0.0 - { - metrics.min_processing_time_ms = processing_time_ms; + // 计算平均处理时间 - 使用增量更新避免重复计算 + let total_events = metrics.event_metrics[index].events_processed; + if total_events > 0 { + let total_events_f64 = total_events as f64; + let old_total = (total_events_f64 - events_processed as f64).max(0.0); + + metrics.average_processing_time_us = if old_total > 0.0 { + (metrics.average_processing_time_us * old_total + + processing_time_us * events_processed as f64) + / total_events_f64 + } else { + processing_time_us + }; + } + + // 更新时间窗口指标 + let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS); + metrics.update_window_metrics(&event_type, now, window_duration); + break; + }, + Err(_) => { + // 如果获取写锁失败，使用指数退避策略 + backoff.snooze(); + continue; + } + } } - if processing_time_ms > metrics.max_processing_time_ms { - metrics.max_processing_time_ms = processing_time_ms; - } - - // 计算平均处理时间 - 使用增量更新避免重复计算 - let total_events = metrics.event_metrics[index].events_processed; - if total_events > 0 { - let total_events_f64 = total_events as f64; - let old_total = (total_events_f64 - events_processed as f64).max(0.0); - - metrics.average_processing_time_ms = if old_total > 0.0 { - (metrics.average_processing_time_ms * old_total - + processing_time_ms * events_processed as f64) - / total_events_f64 - } else { - processing_time_ms - }; - } - - // 更新时间窗口指标 - let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS); - metrics.update_window_metrics(&event_type, now, window_duration); } /// 记录慢处理操作 - pub fn log_slow_processing(&self, processing_time_ms: f64, event_count: usize) { - if processing_time_ms > SLOW_PROCESSING_THRESHOLD_MS { + pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) { + if processing_time_us > SLOW_PROCESSING_THRESHOLD_US { log::warn!( - "{} slow processing: {processing_time_ms}ms for {event_count} events", + "{} slow processing: {processing_time_us}us for {event_count} events", self.stream_name ); } diff --git a/src/streaming/common/mod.rs b/src/streaming/common/mod.rs index bbbc5e1..011bab8 100644 --- a/src/streaming/common/mod.rs +++ b/src/streaming/common/mod.rs @@ -4,6 +4,7 @@ pub mod metrics; pub mod batch; pub mod constants; pub mod subscription; +pub mod event_processor; // 重新导出主要类型 pub use config::*; @@ -11,3 +12,4 @@ pub use metrics::*; pub use batch::*; pub use constants::*; pub use subscription::*; +pub use event_processor::*; \ No newline at end of file diff --git a/src/streaming/common/subscription.rs b/src/streaming/common/subscription.rs index ca29cbb..c0abf16 100644 --- a/src/streaming/common/subscription.rs +++ b/src/streaming/common/subscription.rs @@ -3,7 +3,7 @@ use tokio::task::JoinHandle; /// Subscription handle for managing and stopping subscriptions pub struct SubscriptionHandle { stream_handle: JoinHandle<()>, - event_handle: JoinHandle<()>, + event_handle: Option>, metrics_handle: Option>, } @@ -11,7 +11,7 @@ impl SubscriptionHandle { /// Create a new subscription handle pub fn new( stream_handle: JoinHandle<()>, - event_handle: JoinHandle<()>, + event_handle: Option>, metrics_handle: Option>, ) -> Self { Self { stream_handle, event_handle, metrics_handle } @@ -20,7 +20,9 @@ impl SubscriptionHandle { /// Stop subscription and abort all related tasks pub fn stop(self) { self.stream_handle.abort(); - self.event_handle.abort(); + if let Some(handle) = self.event_handle { + handle.abort(); + } if let Some(handle) = self.metrics_handle { handle.abort(); } @@ -29,7 +31,9 @@ impl SubscriptionHandle { /// Asynchronously wait for all tasks to complete pub async fn join(self) -> Result<(), tokio::task::JoinError> { let _ = self.stream_handle.await; - let _ = self.event_handle.await; + if let Some(handle) = self.event_handle { + let _ = handle.await; + } if let Some(handle) = self.metrics_handle { let _ = handle.await; } diff --git a/src/streaming/event_parser/common/mod.rs b/src/streaming/event_parser/common/mod.rs index 3ecc161..d421559 100755 --- a/src/streaming/event_parser/common/mod.rs +++ b/src/streaming/event_parser/common/mod.rs @@ -2,6 +2,8 @@ pub mod types; pub mod utils; pub mod filter; +pub const EMPTY_ID: &str = ""; + /// 自动生成UnifiedEvent trait实现的宏 #[macro_export] macro_rules! impl_unified_event { @@ -12,6 +14,10 @@ macro_rules! impl_unified_event { &self.metadata.id } + fn clear_id(&mut self) { + self.metadata.id = $crate::streaming::event_parser::common::EMPTY_ID.to_string(); + } + fn event_type(&self) -> $crate::streaming::event_parser::common::types::EventType { self.metadata.event_type.clone() } @@ -24,16 +30,16 @@ macro_rules! impl_unified_event { self.metadata.slot } - fn program_received_time_ms(&self) -> i64 { - self.metadata.program_received_time_ms + fn program_received_time_us(&self) -> i64 { + self.metadata.program_received_time_us } - fn program_handle_time_consuming_ms(&self) -> i64 { - self.metadata.program_handle_time_consuming_ms + fn program_handle_time_consuming_us(&self) -> i64 { + self.metadata.program_handle_time_consuming_us } - fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64) { - self.metadata.program_handle_time_consuming_ms = program_handle_time_consuming_ms; + fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64) { + self.metadata.program_handle_time_consuming_us = program_handle_time_consuming_us; } fn as_any(&self) -> &dyn std::any::Any { @@ -56,8 +62,8 @@ macro_rules! impl_unified_event { } } - fn set_transfer_datas(&mut self, transfer_datas: Vec<$crate::streaming::event_parser::common::types::TransferData>, swap_data: Option<$crate::streaming::event_parser::common::types::SwapData>) { - self.metadata.set_transfer_datas(transfer_datas, swap_data); + fn set_swap_data(&mut self, swap_data: $crate::streaming::event_parser::common::types::SwapData) { + self.metadata.set_swap_data(swap_data); } fn index(&self) -> String { diff --git a/src/streaming/event_parser/common/types.rs b/src/streaming/event_parser/common/types.rs index 9794cc8..37e75e4 100755 --- a/src/streaming/event_parser/common/types.rs +++ b/src/streaming/event_parser/common/types.rs @@ -1,13 +1,14 @@ use borsh::{BorshDeserialize, BorshSerialize}; +use crossbeam_queue::ArrayQueue; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; -use solana_transaction_status::UiInstruction; +use solana_transaction_status::{InnerInstruction, UiInstruction}; use std::{ + fmt, hash::{DefaultHasher, Hash, Hasher}, str::FromStr, sync::Arc, }; -use tokio::sync::Mutex; use crate::{ match_event, @@ -30,7 +31,7 @@ const TRANSFER_DATA_POOL_SIZE: usize = 2000; /// Event metadata object pool pub struct EventMetadataPool { - pool: Arc>>, + pool: Arc>, } impl Default for EventMetadataPool { @@ -41,25 +42,22 @@ impl Default for EventMetadataPool { impl EventMetadataPool { pub fn new() -> Self { - Self { pool: Arc::new(Mutex::new(Vec::with_capacity(EVENT_METADATA_POOL_SIZE))) } + Self { pool: Arc::new(ArrayQueue::new(EVENT_METADATA_POOL_SIZE)) } } - pub async fn acquire(&self) -> Option { - let mut pool = self.pool.lock().await; - pool.pop() + pub fn acquire(&self) -> Option { + self.pool.pop() } - pub async fn release(&self, metadata: EventMetadata) { - let mut pool = self.pool.lock().await; - if pool.len() < EVENT_METADATA_POOL_SIZE { - pool.push(metadata); - } + pub fn release(&self, metadata: EventMetadata) { + // 如果队列已满，push 会失败，但不会阻塞 + let _ = self.pool.push(metadata); } } /// Transfer data object pool pub struct TransferDataPool { - pool: Arc>>, + pool: Arc>, } impl Default for TransferDataPool { @@ -70,19 +68,16 @@ impl Default for TransferDataPool { impl TransferDataPool { pub fn new() -> Self { - Self { pool: Arc::new(Mutex::new(Vec::with_capacity(TRANSFER_DATA_POOL_SIZE))) } + Self { pool: Arc::new(ArrayQueue::new(TRANSFER_DATA_POOL_SIZE)) } } - pub async fn acquire(&self) -> Option { - let mut pool = self.pool.lock().await; - pool.pop() + pub fn acquire(&self) -> Option { + self.pool.pop() } - pub async fn release(&self, transfer_data: TransferData) { - let mut pool = self.pool.lock().await; - if pool.len() < TRANSFER_DATA_POOL_SIZE { - pool.push(transfer_data); - } + pub fn release(&self, transfer_data: TransferData) { + // 如果队列已满，push 会失败，但不会阻塞 + let _ = self.pool.push(transfer_data); } } @@ -199,70 +194,69 @@ pub const ACCOUNT_EVENT_TYPES: &[EventType] = &[ ]; pub const BLOCK_EVENT_TYPES: &[EventType] = &[EventType::BlockMeta]; -impl EventType { - #[allow(clippy::inherent_to_string)] - pub fn to_string(&self) -> String { +impl fmt::Display for EventType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - EventType::PumpSwapBuy => "PumpSwapBuy".to_string(), - EventType::PumpSwapSell => "PumpSwapSell".to_string(), - EventType::PumpSwapCreatePool => "PumpSwapCreatePool".to_string(), - EventType::PumpSwapDeposit => "PumpSwapDeposit".to_string(), - EventType::PumpSwapWithdraw => "PumpSwapWithdraw".to_string(), - EventType::PumpFunCreateToken => "PumpFunCreateToken".to_string(), - EventType::PumpFunBuy => "PumpFunBuy".to_string(), - EventType::PumpFunSell => "PumpFunSell".to_string(), - EventType::PumpFunMigrate => "PumpFunMigrate".to_string(), - EventType::BonkBuyExactIn => "BonkBuyExactIn".to_string(), - EventType::BonkBuyExactOut => "BonkBuyExactOut".to_string(), - EventType::BonkSellExactIn => "BonkSellExactIn".to_string(), - EventType::BonkSellExactOut => "BonkSellExactOut".to_string(), - EventType::BonkInitialize => "BonkInitialize".to_string(), - EventType::BonkInitializeV2 => "BonkInitializeV2".to_string(), - EventType::BonkMigrateToAmm => "BonkMigrateToAmm".to_string(), - EventType::BonkMigrateToCpswap => "BonkMigrateToCpswap".to_string(), - EventType::AccountPumpFunBondingCurve => "AccountPumpFunBondingCurve".to_string(), - EventType::AccountPumpFunGlobal => "AccountPumpFunGlobal".to_string(), - EventType::AccountPumpSwapGlobalConfig => "AccountPumpSwapGlobalConfig".to_string(), - EventType::AccountPumpSwapPool => "AccountPumpSwapPool".to_string(), - EventType::AccountBonkPoolState => "AccountBonkPoolState".to_string(), - EventType::AccountBonkGlobalConfig => "AccountBonkGlobalConfig".to_string(), - EventType::AccountBonkPlatformConfig => "AccountBonkPlatformConfig".to_string(), - EventType::AccountBonkVestingRecord => "AccountBonkVestingRecord".to_string(), - EventType::RaydiumCpmmSwapBaseInput => "RaydiumCpmmSwapBaseInput".to_string(), - EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmmSwapBaseOutput".to_string(), - EventType::RaydiumCpmmDeposit => "RaydiumCpmmDeposit".to_string(), - EventType::RaydiumCpmmInitialize => "RaydiumCpmmInitialize".to_string(), - EventType::RaydiumCpmmWithdraw => "RaydiumCpmmWithdraw".to_string(), - EventType::RaydiumClmmSwap => "RaydiumClmmSwap".to_string(), - EventType::RaydiumClmmSwapV2 => "RaydiumClmmSwapV2".to_string(), - EventType::RaydiumClmmClosePosition => "RaydiumClmmClosePosition".to_string(), + EventType::PumpSwapBuy => write!(f, "PumpSwapBuy"), + EventType::PumpSwapSell => write!(f, "PumpSwapSell"), + EventType::PumpSwapCreatePool => write!(f, "PumpSwapCreatePool"), + EventType::PumpSwapDeposit => write!(f, "PumpSwapDeposit"), + EventType::PumpSwapWithdraw => write!(f, "PumpSwapWithdraw"), + EventType::PumpFunCreateToken => write!(f, "PumpFunCreateToken"), + EventType::PumpFunBuy => write!(f, "PumpFunBuy"), + EventType::PumpFunSell => write!(f, "PumpFunSell"), + EventType::PumpFunMigrate => write!(f, "PumpFunMigrate"), + EventType::BonkBuyExactIn => write!(f, "BonkBuyExactIn"), + EventType::BonkBuyExactOut => write!(f, "BonkBuyExactOut"), + EventType::BonkSellExactIn => write!(f, "BonkSellExactIn"), + EventType::BonkSellExactOut => write!(f, "BonkSellExactOut"), + EventType::BonkInitialize => write!(f, "BonkInitialize"), + EventType::BonkInitializeV2 => write!(f, "BonkInitializeV2"), + EventType::BonkMigrateToAmm => write!(f, "BonkMigrateToAmm"), + EventType::BonkMigrateToCpswap => write!(f, "BonkMigrateToCpswap"), + EventType::RaydiumCpmmSwapBaseInput => write!(f, "RaydiumCpmmSwapBaseInput"), + EventType::RaydiumCpmmSwapBaseOutput => write!(f, "RaydiumCpmmSwapBaseOutput"), + EventType::RaydiumCpmmDeposit => write!(f, "RaydiumCpmmDeposit"), + EventType::RaydiumCpmmInitialize => write!(f, "RaydiumCpmmInitialize"), + EventType::RaydiumCpmmWithdraw => write!(f, "RaydiumCpmmWithdraw"), + EventType::RaydiumClmmSwap => write!(f, "RaydiumClmmSwap"), + EventType::RaydiumClmmSwapV2 => write!(f, "RaydiumClmmSwapV2"), + EventType::RaydiumClmmClosePosition => write!(f, "RaydiumClmmClosePosition"), EventType::RaydiumClmmDecreaseLiquidityV2 => { - "RaydiumClmmDecreaseLiquidityV2".to_string() + write!(f, "RaydiumClmmDecreaseLiquidityV2") } - EventType::RaydiumClmmCreatePool => "RaydiumClmmCreatePool".to_string(), + EventType::RaydiumClmmCreatePool => write!(f, "RaydiumClmmCreatePool"), EventType::RaydiumClmmIncreaseLiquidityV2 => { - "RaydiumClmmIncreaseLiquidityV2".to_string() + write!(f, "RaydiumClmmIncreaseLiquidityV2") } EventType::RaydiumClmmOpenPositionWithToken22Nft => { - "RaydiumClmmOpenPositionWithToken22Nft".to_string() + write!(f, "RaydiumClmmOpenPositionWithToken22Nft") } - EventType::RaydiumClmmOpenPositionV2 => "RaydiumClmmOpenPositionV2".to_string(), - EventType::RaydiumAmmV4SwapBaseIn => "RaydiumAmmV4SwapBaseIn".to_string(), - EventType::RaydiumAmmV4SwapBaseOut => "RaydiumAmmV4SwapBaseOut".to_string(), - EventType::RaydiumAmmV4Deposit => "RaydiumAmmV4Deposit".to_string(), - EventType::RaydiumAmmV4Initialize2 => "RaydiumAmmV4Initialize2".to_string(), - EventType::RaydiumAmmV4Withdraw => "RaydiumAmmV4Withdraw".to_string(), - EventType::RaydiumAmmV4WithdrawPnl => "RaydiumAmmV4WithdrawPnl".to_string(), - EventType::AccountRaydiumAmmV4AmmInfo => "AccountRaydiumAmmV4AmmInfo".to_string(), - EventType::AccountRaydiumClmmAmmConfig => "AccountRaydiumClmmAmmConfig".to_string(), - EventType::AccountRaydiumClmmPoolState => "AccountRaydiumClmmPoolState".to_string(), + EventType::RaydiumClmmOpenPositionV2 => write!(f, "RaydiumClmmOpenPositionV2"), + EventType::RaydiumAmmV4SwapBaseIn => write!(f, "RaydiumAmmV4SwapBaseIn"), + EventType::RaydiumAmmV4SwapBaseOut => write!(f, "RaydiumAmmV4SwapBaseOut"), + EventType::RaydiumAmmV4Deposit => write!(f, "RaydiumAmmV4Deposit"), + EventType::RaydiumAmmV4Initialize2 => write!(f, "RaydiumAmmV4Initialize2"), + EventType::RaydiumAmmV4Withdraw => write!(f, "RaydiumAmmV4Withdraw"), + EventType::RaydiumAmmV4WithdrawPnl => write!(f, "RaydiumAmmV4WithdrawPnl"), + EventType::AccountRaydiumAmmV4AmmInfo => write!(f, "AccountRaydiumAmmV4AmmInfo"), + EventType::AccountPumpSwapGlobalConfig => write!(f, "AccountPumpSwapGlobalConfig"), + EventType::AccountPumpSwapPool => write!(f, "AccountPumpSwapPool"), + EventType::AccountBonkPoolState => write!(f, "AccountBonkPoolState"), + EventType::AccountBonkGlobalConfig => write!(f, "AccountBonkGlobalConfig"), + EventType::AccountBonkPlatformConfig => write!(f, "AccountBonkPlatformConfig"), + EventType::AccountBonkVestingRecord => write!(f, "AccountBonkVestingRecord"), + EventType::AccountPumpFunBondingCurve => write!(f, "AccountPumpFunBondingCurve"), + EventType::AccountPumpFunGlobal => write!(f, "AccountPumpFunGlobal"), + EventType::AccountRaydiumClmmAmmConfig => write!(f, "AccountRaydiumClmmAmmConfig"), + EventType::AccountRaydiumClmmPoolState => write!(f, "AccountRaydiumClmmPoolState"), EventType::AccountRaydiumClmmTickArrayState => { - "AccountRaydiumClmmTickArrayState".to_string() + write!(f, "AccountRaydiumClmmTickArrayState") } - EventType::AccountRaydiumCpmmAmmConfig => "AccountRaydiumCpmmAmmConfig".to_string(), - EventType::AccountRaydiumCpmmPoolState => "AccountRaydiumCpmmPoolState".to_string(), - EventType::BlockMeta => "BlockMeta".to_string(), - EventType::Unknown => "Unknown".to_string(), + EventType::AccountRaydiumCpmmAmmConfig => write!(f, "AccountRaydiumCpmmAmmConfig"), + EventType::AccountRaydiumCpmmPoolState => write!(f, "AccountRaydiumCpmmPoolState"), + EventType::BlockMeta => write!(f, "BlockMeta"), + EventType::Unknown => write!(f, "Unknown"), } } } @@ -345,11 +339,12 @@ pub struct EventMetadata { pub slot: u64, pub block_time: i64, pub block_time_ms: i64, - pub program_received_time_ms: i64, - pub program_handle_time_consuming_ms: i64, + pub program_received_time_us: i64, + pub program_handle_time_consuming_us: i64, pub protocol: ProtocolType, pub event_type: EventType, pub program_id: Pubkey, + #[deprecated(note = "Please use swap_data instead")] pub transfer_datas: Vec, pub swap_data: Option, pub index: String, @@ -367,7 +362,7 @@ impl EventMetadata { event_type: EventType, program_id: Pubkey, index: String, - program_received_time_ms: i64, + program_received_time_us: i64, ) -> Self { Self { id, @@ -375,250 +370,196 @@ impl EventMetadata { slot, block_time, block_time_ms, - program_received_time_ms, - program_handle_time_consuming_ms: 0, + program_received_time_us, + program_handle_time_consuming_us: 0, protocol, event_type, program_id, - transfer_datas: Vec::with_capacity(4), // Pre-allocate capacity + transfer_datas: vec![], swap_data: None, index, } } pub fn set_id(&mut self, id: String) { - let _id = format!("{}-{}-{}", self.signature, self.event_type.to_string(), id); - let mut hasher = DefaultHasher::new(); - _id.hash(&mut hasher); - let hash_value = hasher.finish(); - self.id = format!("{:x}", hash_value); + self.id = format!("{}-{}-{}", self.signature, self.event_type, id); } - pub fn set_transfer_datas( - &mut self, - transfer_datas: Vec, - swap_data: Option, - ) { - self.transfer_datas = transfer_datas; - self.swap_data = swap_data; + pub fn set_swap_data(&mut self, swap_data: SwapData) { + self.swap_data = Some(swap_data); } /// Recycle EventMetadata to object pool - pub async fn recycle(self) { - EVENT_METADATA_POOL.release(self).await; + pub fn recycle(self) { + EVENT_METADATA_POOL.release(self); } } -/// Parse token transfer data from next instructions -pub fn parse_transfer_datas_from_next_instructions( - event: Box, - inner_instruction: &solana_transaction_status::UiInnerInstructions, - current_index: i8, - accounts: &[Pubkey], -) -> (Vec, Option) { - let mut transfer_datas = vec![]; - // Get the next two instructions after the current instruction - let next_instructions: Vec<&UiInstruction> = - inner_instruction.instructions.iter().skip((current_index + 1) as usize).collect(); - - let system_programs = vec![ - // Token Program +lazy_static::lazy_static! { + static ref SOL_MINT: Pubkey = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap(); + static ref SYSTEM_PROGRAMS: [Pubkey; 3] = [ Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), - // Token 2022 Program Pubkey::from_str("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb").unwrap(), - // System Program Pubkey::from_str("11111111111111111111111111111111").unwrap(), ]; - for instruction in next_instructions { - if let UiInstruction::Compiled(compiled) = instruction { - if !system_programs.contains(&accounts[compiled.program_id_index as usize]) { - break; - } - if let Ok(data) = bs58::decode(compiled.data.clone()).into_vec() { - // Token Program: transferChecked - // Token 2022 Program: transferChecked - if data[0] == 12 { - let account_pubkeys: Vec = - compiled.accounts.iter().map(|a| accounts[*a as usize]).collect(); - if account_pubkeys.len() < 4 { - continue; - } - let (source, mint, destination, authority) = ( - account_pubkeys[0], - account_pubkeys[1], - account_pubkeys[2], - account_pubkeys[3], - ); - let amount = u64::from_le_bytes(data[1..9].try_into().unwrap()); - let decimals = data[9]; - let token_program = accounts[compiled.program_id_index as usize]; - transfer_datas.push(TransferData { - amount, - decimals: Some(decimals), - mint: Some(mint), - source, - destination, - authority: Some(authority), - token_program, - }); - } - // Token Program: transfer - else if data[0] == 3 { - let account_pubkeys: Vec = - compiled.accounts.iter().map(|a| accounts[*a as usize]).collect(); - if account_pubkeys.len() < 3 { - continue; - } - let (source, destination, authority) = - (account_pubkeys[0], account_pubkeys[1], account_pubkeys[2]); - let amount = u64::from_le_bytes(data[1..9].try_into().unwrap()); - let token_program = accounts[compiled.program_id_index as usize]; - transfer_datas.push(TransferData { - amount, - decimals: None, - mint: None, - source, - destination, - authority: Some(authority), - token_program, - }); - } - //System Program: transfer - else if data[0] == 2 { - let account_pubkeys: Vec = - compiled.accounts.iter().map(|a| accounts[*a as usize]).collect(); - if account_pubkeys.len() < 2 { - continue; - } - let (source, destination) = (account_pubkeys[0], account_pubkeys[1]); - let amount = u64::from_le_bytes(data[4..12].try_into().unwrap()); - let token_program = accounts[compiled.program_id_index as usize]; - transfer_datas.push(TransferData { - amount, - decimals: None, - mint: None, - source, - destination, - authority: None, - token_program, - }); - } - } - } - } - let mut swap_data: SwapData = SwapData { +} + +/// Parse token transfer data from next instructions +pub fn parse_swap_data_from_next_instructions( + event: Box, + inner_instruction: &solana_transaction_status::InnerInstructions, + current_index: i8, + accounts: &[Pubkey], +) -> Option { + let mut swap_data = SwapData { from_mint: Pubkey::default(), to_mint: Pubkey::default(), from_amount: 0, to_amount: 0, description: None, }; - let sol_mint = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap(); - if transfer_datas.len() > 0 { - let mut user: Option = None; - let mut from_mint: Option = None; - let mut to_mint: Option = None; - let mut user_from_token: Option = None; - let mut user_to_token: Option = None; - let mut from_vault: Option = None; - let mut to_vault: Option = None; - match_event!(event, { - BonkTradeEvent => |e: BonkTradeEvent| { - user = Some(e.payer); - from_mint = Some(e.base_token_mint); - to_mint = Some(e.quote_token_mint); - user_from_token = Some(e.user_base_token); - user_to_token = Some(e.user_quote_token); - from_vault = Some(e.base_vault); - to_vault = Some(e.quote_vault); - }, - PumpFunTradeEvent => |e: PumpFunTradeEvent| { - swap_data.from_mint = if e.is_buy { - sol_mint - } else { - e.mint - }; - swap_data.to_mint = if e.is_buy { - e.mint - } else { - sol_mint - }; - }, - PumpSwapBuyEvent => |e: PumpSwapBuyEvent| { - swap_data.from_mint = e.quote_mint; - swap_data.to_mint = e.base_mint; - }, - PumpSwapSellEvent => |e: PumpSwapSellEvent| { - swap_data.from_mint = e.base_mint; - swap_data.to_mint = e.quote_mint; - }, - RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| { - user = Some(e.payer); - from_mint = Some(e.input_token_mint); - to_mint = Some(e.output_token_mint); - user_from_token = Some(e.input_token_account); - user_to_token = Some(e.output_token_account); - from_vault = Some(e.input_vault); - to_vault = Some(e.output_vault); - }, - RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| { - user = Some(e.payer); - swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string()); - user_from_token = Some(e.input_token_account); - user_to_token = Some(e.output_token_account); - from_vault = Some(e.input_vault); - to_vault = Some(e.output_vault); - }, - RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| { - user = Some(e.payer); - from_mint = Some(e.input_vault_mint); - to_mint = Some(e.output_vault_mint); - user_from_token = Some(e.input_token_account); - user_to_token = Some(e.output_token_account); - from_vault = Some(e.input_vault); - to_vault = Some(e.output_vault); - }, - RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| { - user = Some(e.user_source_owner); - swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string()); - user_from_token = Some(e.user_source_token_account); - user_to_token = Some(e.user_destination_token_account); - from_vault = Some(e.pool_pc_token_account); - to_vault = Some(e.pool_coin_token_account); - }, - }); - for transfer_data in transfer_datas.clone() { - if transfer_data.source == user_to_token.unwrap_or_default() - && transfer_data.destination == to_vault.unwrap_or_default() - { - swap_data.from_mint = to_mint.unwrap_or_default(); - swap_data.from_amount = transfer_data.amount; - } else if transfer_data.source == from_vault.unwrap_or_default() - && transfer_data.destination == user_from_token.unwrap_or_default() - { - swap_data.to_mint = from_mint.unwrap_or_default(); - swap_data.to_amount = transfer_data.amount; - } else if transfer_data.source == user_from_token.unwrap_or_default() - && transfer_data.destination == from_vault.unwrap_or_default() - { - swap_data.from_mint = from_mint.unwrap_or_default(); - swap_data.from_amount = transfer_data.amount; - } else if transfer_data.source == to_vault.unwrap_or_default() - && transfer_data.destination == user_to_token.unwrap_or_default() - { - swap_data.to_mint = to_mint.unwrap_or_default(); - swap_data.to_amount = transfer_data.amount; + // 先根据 event 取出关键信息 + let mut user: Option = None; + let mut from_mint: Option = None; + let mut to_mint: Option = None; + let mut user_from_token: Option = None; + let mut user_to_token: Option = None; + let mut from_vault: Option = None; + let mut to_vault: Option = None; + + match_event!(event, { + BonkTradeEvent => |e: BonkTradeEvent| { + user = Some(e.payer); + from_mint = Some(e.base_token_mint); + to_mint = Some(e.quote_token_mint); + user_from_token = Some(e.user_base_token); + user_to_token = Some(e.user_quote_token); + from_vault = Some(e.base_vault); + to_vault = Some(e.quote_vault); + }, + PumpFunTradeEvent => |e: PumpFunTradeEvent| { + swap_data.from_mint = if e.is_buy { *SOL_MINT } else { e.mint }; + swap_data.to_mint = if e.is_buy { e.mint } else { *SOL_MINT }; + }, + PumpSwapBuyEvent => |e: PumpSwapBuyEvent| { + swap_data.from_mint = e.quote_mint; + swap_data.to_mint = e.base_mint; + }, + PumpSwapSellEvent => |e: PumpSwapSellEvent| { + swap_data.from_mint = e.base_mint; + swap_data.to_mint = e.quote_mint; + }, + RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| { + user = Some(e.payer); + from_mint = Some(e.input_token_mint); + to_mint = Some(e.output_token_mint); + user_from_token = Some(e.input_token_account); + user_to_token = Some(e.output_token_account); + from_vault = Some(e.input_vault); + to_vault = Some(e.output_vault); + }, + RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| { + user = Some(e.payer); + swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string()); + user_from_token = Some(e.input_token_account); + user_to_token = Some(e.output_token_account); + from_vault = Some(e.input_vault); + to_vault = Some(e.output_vault); + }, + RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| { + user = Some(e.payer); + from_mint = Some(e.input_vault_mint); + to_mint = Some(e.output_vault_mint); + user_from_token = Some(e.input_token_account); + user_to_token = Some(e.output_token_account); + from_vault = Some(e.input_vault); + to_vault = Some(e.output_vault); + }, + RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| { + user = Some(e.user_source_owner); + swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string()); + user_from_token = Some(e.user_source_token_account); + user_to_token = Some(e.user_destination_token_account); + from_vault = Some(e.pool_pc_token_account); + to_vault = Some(e.pool_coin_token_account); + }, + }); + + let user_to_token = user_to_token.unwrap_or_default(); + let user_from_token = user_from_token.unwrap_or_default(); + let to_vault = to_vault.unwrap_or_default(); + let from_vault = from_vault.unwrap_or_default(); + let to_mint = to_mint.unwrap_or_default(); + let from_mint = from_mint.unwrap_or_default(); + + // 单次循环完成提取和判断 + for instruction in inner_instruction.instructions.iter().skip((current_index + 1) as usize) { + let compiled = &instruction.instruction; + let program_id = accounts[compiled.program_id_index as usize]; + if !SYSTEM_PROGRAMS.contains(&program_id) { + break; + } + let data = &compiled.data; + let get_pubkey = |i: usize| accounts[compiled.accounts[i] as usize]; + let (source, destination, amount) = match data[0] { + 12 if compiled.accounts.len() >= 4 => { + let amt = u64::from_le_bytes(data[1..9].try_into().unwrap()); + (get_pubkey(0), get_pubkey(2), amt) } + 3 if compiled.accounts.len() >= 3 => { + let amt = u64::from_le_bytes(data[1..9].try_into().unwrap()); + (get_pubkey(0), get_pubkey(1), amt) + } + 2 if compiled.accounts.len() >= 2 => { + let amt = u64::from_le_bytes(data[4..12].try_into().unwrap()); + (get_pubkey(0), get_pubkey(1), amt) + } + _ => continue, + }; + + match (source, destination) { + (s, d) if s == user_to_token && d == to_vault => { + swap_data.from_mint = to_mint; + swap_data.from_amount = amount; + } + (s, d) if s == from_vault && d == user_from_token => { + swap_data.to_mint = from_mint; + swap_data.to_amount = amount; + } + (s, d) if s == user_from_token && d == from_vault => { + swap_data.from_mint = from_mint; + swap_data.from_amount = amount; + } + (s, d) if s == to_vault && d == user_to_token => { + swap_data.to_mint = to_mint; + swap_data.to_amount = amount; + } + (s, d) if s == user_from_token && d == to_vault => { + swap_data.from_mint = from_mint; + swap_data.from_amount = amount; + } + (s, d) if s == from_vault && d == user_to_token => { + swap_data.to_mint = to_mint; + swap_data.to_amount = amount; + } + _ => {} + } + if swap_data.from_mint != Pubkey::default() && swap_data.to_mint != Pubkey::default() { + break; + } + if swap_data.from_amount != 0 && swap_data.to_amount != 0 { + break; } } + if swap_data.from_mint != Pubkey::default() || swap_data.to_mint != Pubkey::default() || swap_data.from_amount != 0 || swap_data.to_amount != 0 { - (transfer_datas, Some(swap_data)) + Some(swap_data) } else { - (transfer_datas, None) + None } } diff --git a/src/streaming/event_parser/common/utils.rs b/src/streaming/event_parser/common/utils.rs index 2d372b2..80458d4 100755 --- a/src/streaming/event_parser/common/utils.rs +++ b/src/streaming/event_parser/common/utils.rs @@ -1,5 +1,3 @@ -use base64::engine::general_purpose; -use base64::Engine; use std::time::{SystemTime, UNIX_EPOCH}; /// 获取当前时间戳 @@ -7,16 +5,6 @@ pub fn current_timestamp() -> i64 { SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs() as i64 } -/// 从base64字符串解码数据 -pub fn decode_base64(data: &str) -> Result, base64::DecodeError> { - general_purpose::STANDARD.decode(data) -} - -/// 将数据编码为base64字符串 -pub fn encode_base64(data: &[u8]) -> String { - general_purpose::STANDARD.encode(data) -} - /// 从字节数组中提取鉴别器和剩余数据 pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8])> { if data.len() < length { diff --git a/src/streaming/event_parser/core/account_event_parser.rs b/src/streaming/event_parser/core/account_event_parser.rs index 7e95698..030266a 100644 --- a/src/streaming/event_parser/core/account_event_parser.rs +++ b/src/streaming/event_parser/core/account_event_parser.rs @@ -158,12 +158,11 @@ impl AccountEventParser { pub fn parse_account_event( protocols: Vec, account: AccountPretty, - program_received_time_ms: i64, event_type_filter: Option, ) -> Option> { let configs = Self::configs(protocols, event_type_filter); for config in configs { - if account.owner == config.program_id.to_string() + if account.owner == config.program_id && account.data[..config.account_discriminator.len()] == *config.account_discriminator { @@ -171,17 +170,17 @@ impl AccountEventParser { &account, EventMetadata { slot: account.slot, - signature: account.signature.clone(), + signature: account.signature.to_string(), protocol: config.protocol_type, event_type: config.event_type, program_id: config.program_id, - program_received_time_ms, + program_received_time_us: account.program_received_time_us, ..Default::default() }, ); if let Some(mut event) = event { - event.set_program_handle_time_consuming_ms( - chrono::Utc::now().timestamp_millis() - program_received_time_ms, + event.set_program_handle_time_consuming_us( + chrono::Utc::now().timestamp_micros() - account.program_received_time_us, ); return Some(event); } diff --git a/src/streaming/event_parser/core/common_event_parser.rs b/src/streaming/event_parser/core/common_event_parser.rs index 8442446..6ec37ae 100644 --- a/src/streaming/event_parser/core/common_event_parser.rs +++ b/src/streaming/event_parser/core/common_event_parser.rs @@ -8,8 +8,17 @@ impl CommonEventParser { slot: u64, block_hash: &str, block_time_ms: i64, + program_received_time_us: i64, ) -> Box { - let block_meta_event = BlockMetaEvent::new(slot, block_hash.to_string(), block_time_ms); + let mut block_meta_event = BlockMetaEvent::new( + slot, + block_hash.to_string(), + block_time_ms, + program_received_time_us, + ); + block_meta_event.set_program_handle_time_consuming_us( + chrono::Utc::now().timestamp_micros() - program_received_time_us, + ); Box::new(block_meta_event) } } diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index a6d8574..d29e52e 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -1,17 +1,14 @@ use anyhow::Result; use prost_types::Timestamp; +use solana_sdk::signature::Signature; use solana_sdk::{ instruction::CompiledInstruction, pubkey::Pubkey, transaction::VersionedTransaction, }; -use solana_transaction_status::{ - EncodedTransactionWithStatusMeta, UiCompiledInstruction, UiInnerInstructions, UiInstruction, -}; +use solana_transaction_status::{InnerInstructions, TransactionWithStatusMeta}; +use std::collections::HashMap; use std::fmt::Debug; -use std::{collections::HashMap, str::FromStr}; -use crate::streaming::event_parser::common::{ - parse_transfer_datas_from_next_instructions, SwapData, TransferData, -}; +use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData}; use crate::streaming::event_parser::protocols::pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}; use crate::streaming::event_parser::{ common::{utils::*, EventMetadata, EventType, ProtocolType}, @@ -20,12 +17,16 @@ use crate::streaming::event_parser::{ pumpfun::{PumpFunCreateTokenEvent, PumpFunTradeEvent}, }, }; +use crate::streaming::shred::MetricsEventType; /// Unified Event Interface - All protocol events must implement this trait pub trait UnifiedEvent: Debug + Send + Sync { /// Get event ID fn id(&self) -> &str; + /// Set event ID + fn clear_id(&mut self); + /// Get event type fn event_type(&self) -> EventType; @@ -36,13 +37,13 @@ pub trait UnifiedEvent: Debug + Send + Sync { fn slot(&self) -> u64; /// Get program received timestamp (milliseconds) - fn program_received_time_ms(&self) -> i64; + fn program_received_time_us(&self) -> i64; /// Processing time consumption (milliseconds) - fn program_handle_time_consuming_ms(&self) -> i64; + fn program_handle_time_consuming_us(&self) -> i64; /// Set processing time consumption (milliseconds) - fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64); + fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64); /// Convert event to Any for downcasting fn as_any(&self) -> &dyn std::any::Any; @@ -58,12 +59,8 @@ pub trait UnifiedEvent: Debug + Send + Sync { // Default implementation: no merging operation } - /// Set transfer datas - fn set_transfer_datas( - &mut self, - transfer_datas: Vec, - swap_data: Option, - ); + /// Set swap data + fn set_swap_data(&mut self, swap_data: SwapData); /// Get index fn index(&self) -> String; @@ -80,11 +77,11 @@ pub trait EventParser: Send + Sync { #[allow(clippy::too_many_arguments)] fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec>; @@ -94,10 +91,10 @@ pub trait EventParser: Send + Sync { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec>; @@ -106,12 +103,12 @@ pub trait EventParser: Send + Sync { async fn parse_instruction_events_from_versioned_transaction( &self, transaction: &VersionedTransaction, - signature: &str, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, accounts: &[Pubkey], - inner_instructions: &[UiInnerInstructions], + inner_instructions: &[InnerInstructions], ) -> Result>> { // 预分配容量，避免动态扩容 let mut instruction_events = Vec::with_capacity(16); @@ -140,7 +137,7 @@ pub trait EventParser: Send + Sync { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, format!("{index}"), ) .await @@ -152,14 +149,15 @@ pub trait EventParser: Send + Sync { }) { events.iter_mut().for_each(|event| { - let (transfer_datas, swap_data) = - parse_transfer_datas_from_next_instructions( - event.clone_boxed(), - inn, - -1_i8, - &accounts, - ); - event.set_transfer_datas(transfer_datas, swap_data); + let swap_data = parse_swap_data_from_next_instructions( + event.clone_boxed(), + inn, + -1_i8, + &accounts, + ); + if let Some(swap_data) = swap_data { + event.set_swap_data(swap_data); + } }); } instruction_events.extend(events); @@ -175,10 +173,10 @@ pub trait EventParser: Send + Sync { async fn parse_versioned_transaction( &self, versioned_tx: &VersionedTransaction, - signature: &str, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, bot_wallet: Option, ) -> Result>> { let accounts: Vec = versioned_tx.message.static_account_keys().to_vec(); @@ -188,7 +186,7 @@ pub trait EventParser: Send + Sync { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, &accounts, &[], ) @@ -199,146 +197,108 @@ pub trait EventParser: Send + Sync { async fn parse_transaction( &self, - tx: EncodedTransactionWithStatusMeta, - signature: &str, + tx: TransactionWithStatusMeta, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, bot_wallet: Option, ) -> Result>> { - // TODO: bug - 待优化 - // // 生成缓存键 - // let cache_key = format!("{}_{}_{}", signature, slot.unwrap_or(0), program_received_time_ms); - - // // 尝试从缓存获取 - // if let Some(cached_events) = PARSE_CACHE.get(&cache_key).await { - // return Ok(cached_events); - // } - - let transaction = tx.transaction; - // 检查交易元数据 - let meta = - tx.meta.as_ref().ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?; + let versioned_tx = tx.get_transaction(); + let meta = tx.get_status_meta(); let mut address_table_lookups: Vec = vec![]; - let mut inner_instructions: Vec = vec![]; - if meta.err.is_none() { - // 正确处理OptionSerializer类型 - if let solana_transaction_status::option_serializer::OptionSerializer::Some( - meta_inner_instructions, - ) = &meta.inner_instructions - { - inner_instructions = meta_inner_instructions.clone(); + let mut inner_instructions: Vec = vec![]; + if let Some(meta) = meta { + inner_instructions = meta.inner_instructions.unwrap_or_default(); + for loopup in meta.loaded_addresses.writable { + address_table_lookups.push(loopup); } - if let solana_transaction_status::option_serializer::OptionSerializer::Some( - loaded_addresses, - ) = &meta.loaded_addresses - { - for lookup in &loaded_addresses.writable { - if let Ok(pubkey) = Pubkey::from_str(lookup) { - address_table_lookups.push(pubkey); - } - } - for lookup in &loaded_addresses.readonly { - if let Ok(pubkey) = Pubkey::from_str(lookup) { - address_table_lookups.push(pubkey); - } - } + for loopup in meta.loaded_addresses.readonly { + address_table_lookups.push(loopup); } } let mut accounts: Vec = vec![]; // 预分配容量，避免动态扩容 - let mut instruction_events = Vec::with_capacity(16); + let mut instruction_events: Vec> = Vec::with_capacity(16); // 解析指令事件 - if let Some(versioned_tx) = transaction.decode() { - accounts = versioned_tx.message.static_account_keys().to_vec(); - accounts.extend(address_table_lookups.clone()); + accounts = versioned_tx.message.static_account_keys().to_vec(); + accounts.extend(address_table_lookups.clone()); - instruction_events = self - .parse_instruction_events_from_versioned_transaction( - &versioned_tx, - signature, - slot, - block_time, - program_received_time_ms, - &accounts, - &inner_instructions, - ) - .await - .unwrap_or_else(|_e| vec![]); - } else { - accounts.extend(address_table_lookups.clone()); - } + instruction_events = self + .parse_instruction_events_from_versioned_transaction( + &versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + &accounts, + &inner_instructions, + ) + .await + .unwrap_or_else(|_e| vec![]); // 解析内联指令事件 // 预分配容量，避免动态扩容 - let mut inner_instruction_events = Vec::with_capacity(8); + let mut inner_instruction_events: Vec> = Vec::with_capacity(8); // 检查交易是否成功 - if meta.err.is_none() { - for inner_instruction in inner_instructions { - for (index, instruction) in inner_instruction.instructions.iter().enumerate() { - if let UiInstruction::Compiled(compiled) = instruction { - // 解析嵌套指令 - let compiled_instruction = CompiledInstruction { - program_id_index: compiled.program_id_index, - accounts: compiled.accounts.clone(), - data: bs58::decode(compiled.data.clone()) - .into_vec() - .unwrap_or_else(|_| vec![]), - }; - if let Ok(mut events) = self - .parse_instruction( - &compiled_instruction, + for inner_instruction in inner_instructions { + for (index, instruction) in inner_instruction.instructions.iter().enumerate() { + // 解析嵌套指令 + let compiled_instruction = instruction.instruction.clone(); + if let Ok(mut events) = self + .parse_instruction( + &compiled_instruction, + &accounts, + signature, + slot, + block_time, + program_received_time_us, + format!("{}.{}", inner_instruction.index, index), + ) + .await + { + if !events.is_empty() { + events.iter_mut().for_each(|event| { + let swap_data = parse_swap_data_from_next_instructions( + event.clone_boxed(), + &inner_instruction, + index as i8, &accounts, - signature, - slot, - block_time, - program_received_time_ms, - format!("{}.{}", inner_instruction.index, index), - ) - .await - { - if !events.is_empty() { - events.iter_mut().for_each(|event| { - let (transfer_datas, swap_data) = - parse_transfer_datas_from_next_instructions( - event.clone_boxed(), - &inner_instruction, - index as i8, - &accounts, - ); - event.set_transfer_datas(transfer_datas, swap_data); - }); - instruction_events.extend(events); + ); + if let Some(swap_data) = swap_data { + event.set_swap_data(swap_data); } - } - if let Ok(mut events) = self - .parse_inner_instruction( - compiled, - signature, - slot, - block_time, - program_received_time_ms, - format!("{}.{}", inner_instruction.index, index), - ) - .await - { - if !events.is_empty() { - events.iter_mut().for_each(|event| { - let (transfer_datas, swap_data) = - parse_transfer_datas_from_next_instructions( - event.clone_boxed(), - &inner_instruction, - index as i8, - &accounts, - ); - event.set_transfer_datas(transfer_datas, swap_data); - }); - inner_instruction_events.extend(events); + }); + instruction_events.extend(events); + } + } + if let Ok(mut events) = self + .parse_inner_instruction( + &compiled_instruction, + signature, + slot, + block_time, + program_received_time_us, + format!("{}.{}", inner_instruction.index, index), + ) + .await + { + if !events.is_empty() { + events.iter_mut().for_each(|event| { + let swap_data = parse_swap_data_from_next_instructions( + event.clone_boxed(), + &inner_instruction, + index as i8, + &accounts, + ); + if let Some(swap_data) = swap_data { + event.set_swap_data(swap_data); } - } + }); + inner_instruction_events.extend(events); } } } @@ -386,9 +346,6 @@ pub trait EventParser: Send + Sync { let result = self.process_events(instruction_events, bot_wallet); - // 缓存结果 - // PARSE_CACHE.set(cache_key, result.clone()).await; - Ok(result) } @@ -397,7 +354,6 @@ pub trait EventParser: Send + Sync { mut events: Vec>, bot_wallet: Option, ) -> Vec> { - let start_time = std::time::Instant::now(); let mut dev_address = vec![]; let mut bonk_dev_address = None; for event in &mut events { @@ -458,18 +414,7 @@ pub trait EventParser: Send + Sync { trade_info.is_dev_create_token_trade = false; } } - let now = chrono::Utc::now().timestamp_millis(); - event.set_program_handle_time_consuming_ms(now - event.program_received_time_ms()); - } - - // 记录处理时间 - let processing_time = start_time.elapsed(); - if processing_time.as_millis() > 10 { - log::warn!( - "Event processing took {}ms for {} events", - processing_time.as_millis(), - events.len() - ); + event.clear_id(); } events @@ -477,11 +422,11 @@ pub trait EventParser: Send + Sync { async fn parse_inner_instruction( &self, - instruction: &UiCompiledInstruction, - signature: &str, + instruction: &CompiledInstruction, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Result>> { let slot = slot.unwrap_or(0); @@ -490,7 +435,7 @@ pub trait EventParser: Send + Sync { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ); Ok(events) @@ -501,10 +446,10 @@ pub trait EventParser: Send + Sync { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: Option, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Result>> { let slot = slot.unwrap_or(0); @@ -514,7 +459,7 @@ pub trait EventParser: Send + Sync { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ); Ok(events) @@ -588,10 +533,10 @@ impl GenericEventParser { &self, config: &GenericEventParseConfig, data: &[u8], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Option> { if let Some(parser) = config.inner_instruction_parser { @@ -607,7 +552,7 @@ impl GenericEventParser { config.event_type.clone(), config.program_id, index, - program_received_time_ms, + program_received_time_us, ); parser(data, metadata) } else { @@ -622,10 +567,10 @@ impl GenericEventParser { config: &GenericEventParseConfig, data: &[u8], account_pubkeys: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Option> { if let Some(parser) = config.instruction_parser { @@ -641,7 +586,7 @@ impl GenericEventParser { config.event_type.clone(), config.program_id, index, - program_received_time_ms, + program_received_time_us, ); parser(data, account_pubkeys, metadata) } else { @@ -662,16 +607,14 @@ impl EventParser for GenericEventParser { #[allow(clippy::too_many_arguments)] fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { - let inner_instruction_data = inner_instruction.data.clone(); - let inner_instruction_data_decoded = - bs58::decode(inner_instruction_data).into_vec().unwrap_or_else(|_| vec![]); + let inner_instruction_data_decoded = inner_instruction.data.clone(); if inner_instruction_data_decoded.len() < 16 { return Vec::new(); } @@ -688,7 +631,7 @@ impl EventParser for GenericEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index.clone(), ) { events.push(event); @@ -705,10 +648,10 @@ impl EventParser for GenericEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { let program_id = accounts[instruction.program_id_index as usize]; @@ -741,7 +684,7 @@ impl EventParser for GenericEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index.clone(), ) { events.push(event); diff --git a/src/streaming/event_parser/protocols/block/block_meta_event.rs b/src/streaming/event_parser/protocols/block/block_meta_event.rs index f4bee6d..2987246 100644 --- a/src/streaming/event_parser/protocols/block/block_meta_event.rs +++ b/src/streaming/event_parser/protocols/block/block_meta_event.rs @@ -13,7 +13,12 @@ pub struct BlockMetaEvent { } impl BlockMetaEvent { - pub fn new(slot: u64, block_hash: String, block_time_ms: i64) -> Self { + pub fn new( + slot: u64, + block_hash: String, + block_time_ms: i64, + program_received_time_us: i64, + ) -> Self { let metadata = EventMetadata::new( format!("block_{}_{}", slot, block_hash), "".to_string(), @@ -24,7 +29,7 @@ impl BlockMetaEvent { EventType::BlockMeta, solana_sdk::pubkey::Pubkey::default(), "".to_string(), - chrono::Utc::now().timestamp_millis(), + program_received_time_us, ); Self { metadata, slot, block_hash } } diff --git a/src/streaming/event_parser/protocols/bonk/parser.rs b/src/streaming/event_parser/protocols/bonk/parser.rs index 056b39d..a03a38e 100755 --- a/src/streaming/event_parser/protocols/bonk/parser.rs +++ b/src/streaming/event_parser/protocols/bonk/parser.rs @@ -1,14 +1,16 @@ use std::collections::HashMap; use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; +use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; use crate::streaming::event_parser::{ common::{utils::*, EventMetadata, EventType, ProtocolType}, core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, protocols::bonk::{ - bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, VestingParams + bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, + BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, + ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, + VestingParams, }, }; @@ -611,11 +613,11 @@ impl EventParser for BonkEventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -623,7 +625,7 @@ impl EventParser for BonkEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -632,10 +634,10 @@ impl EventParser for BonkEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -644,7 +646,7 @@ impl EventParser for BonkEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/event_parser/protocols/mutil/parser.rs b/src/streaming/event_parser/protocols/mutil/parser.rs index af3902d..9748124 100755 --- a/src/streaming/event_parser/protocols/mutil/parser.rs +++ b/src/streaming/event_parser/protocols/mutil/parser.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; use prost_types::Timestamp; +use solana_sdk::signature::Signature; use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::{ @@ -23,18 +23,38 @@ impl MutilEventParser { // Merge inner_instruction_configs, append configurations to existing Vec for (key, configs) in parse.inner_instruction_configs() { - let filtered_configs: Vec = configs.into_iter().filter(|config| { - event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true) - }).collect(); - inner.inner_instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs); + let filtered_configs: Vec = configs + .into_iter() + .filter(|config| { + event_type_filter + .as_ref() + .map(|filter| filter.include.contains(&config.event_type)) + .unwrap_or(true) + }) + .collect(); + inner + .inner_instruction_configs + .entry(key) + .or_insert_with(Vec::new) + .extend(filtered_configs); } // Merge instruction_configs, append configurations to existing Vec for (key, configs) in parse.instruction_configs() { - let filtered_configs: Vec = configs.into_iter().filter(|config| { - event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true) - }).collect(); - inner.instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs); + let filtered_configs: Vec = configs + .into_iter() + .filter(|config| { + event_type_filter + .as_ref() + .map(|filter| filter.include.contains(&config.event_type)) + .unwrap_or(true) + }) + .collect(); + inner + .instruction_configs + .entry(key) + .or_insert_with(Vec::new) + .extend(filtered_configs); } // Append program_ids (this is already appending) @@ -54,11 +74,11 @@ impl EventParser for MutilEventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -66,7 +86,7 @@ impl EventParser for MutilEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -75,10 +95,10 @@ impl EventParser for MutilEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -87,7 +107,7 @@ impl EventParser for MutilEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/event_parser/protocols/pumpfun/parser.rs b/src/streaming/event_parser/protocols/pumpfun/parser.rs index 4faaf01..e907c11 100755 --- a/src/streaming/event_parser/protocols/pumpfun/parser.rs +++ b/src/streaming/event_parser/protocols/pumpfun/parser.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; +use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; use crate::streaming::event_parser::{ common::{EventMetadata, EventType, ProtocolType}, @@ -295,11 +294,11 @@ impl EventParser for PumpFunEventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -307,7 +306,7 @@ impl EventParser for PumpFunEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -316,10 +315,10 @@ impl EventParser for PumpFunEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -328,7 +327,7 @@ impl EventParser for PumpFunEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/event_parser/protocols/pumpswap/parser.rs b/src/streaming/event_parser/protocols/pumpswap/parser.rs index ec84ad1..6bd9b57 100755 --- a/src/streaming/event_parser/protocols/pumpswap/parser.rs +++ b/src/streaming/event_parser/protocols/pumpswap/parser.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; +use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; use crate::streaming::event_parser::{ common::{read_u64_le, EventMetadata, EventType, ProtocolType}, @@ -385,11 +384,11 @@ impl EventParser for PumpSwapEventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -397,7 +396,7 @@ impl EventParser for PumpSwapEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -406,10 +405,10 @@ impl EventParser for PumpSwapEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -418,7 +417,7 @@ impl EventParser for PumpSwapEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs index ba4a5cf..593e1af 100755 --- a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; +use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; use crate::streaming::event_parser::{ common::{read_u64_le, EventMetadata, EventType, ProtocolType}, @@ -387,11 +386,11 @@ impl EventParser for RaydiumAmmV4EventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -399,7 +398,7 @@ impl EventParser for RaydiumAmmV4EventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -408,10 +407,10 @@ impl EventParser for RaydiumAmmV4EventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -420,7 +419,7 @@ impl EventParser for RaydiumAmmV4EventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs index f06f238..aef8ba9 100755 --- a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; +use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; use crate::streaming::event_parser::{ common::{ @@ -428,11 +427,11 @@ impl EventParser for RaydiumClmmEventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -440,7 +439,7 @@ impl EventParser for RaydiumClmmEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -449,10 +448,10 @@ impl EventParser for RaydiumClmmEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -461,7 +460,7 @@ impl EventParser for RaydiumClmmEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs index fdc453a..6dfdd6b 100755 --- a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; -use solana_transaction_status::UiCompiledInstruction; +use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; use crate::streaming::event_parser::{ common::{read_u64_le, EventMetadata, EventType, ProtocolType}, @@ -278,11 +277,11 @@ impl EventParser for RaydiumCpmmEventParser { } fn parse_events_from_inner_instruction( &self, - inner_instruction: &UiCompiledInstruction, - signature: &str, + inner_instruction: &CompiledInstruction, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_inner_instruction( @@ -290,7 +289,7 @@ impl EventParser for RaydiumCpmmEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } @@ -299,10 +298,10 @@ impl EventParser for RaydiumCpmmEventParser { &self, instruction: &CompiledInstruction, accounts: &[Pubkey], - signature: &str, + signature: Signature, slot: u64, block_time: Option, - program_received_time_ms: i64, + program_received_time_us: i64, index: String, ) -> Vec> { self.inner.parse_events_from_instruction( @@ -311,7 +310,7 @@ impl EventParser for RaydiumCpmmEventParser { signature, slot, block_time, - program_received_time_ms, + program_received_time_us, index, ) } diff --git a/src/streaming/grpc/event_processor.rs b/src/streaming/grpc/event_processor.rs deleted file mode 100644 index 10454fe..0000000 --- a/src/streaming/grpc/event_processor.rs +++ /dev/null @@ -1,290 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use solana_sdk::pubkey::Pubkey; - -use super::types::EventPretty; -use crate::common::AnyResult; -use crate::streaming::common::{ - EventBatchProcessor as EventBatchCollector, MetricsEventType, MetricsManager, - StreamClientConfig as ClientConfig, -}; -use crate::streaming::event_parser::common::filter::EventTypeFilter; -use crate::streaming::event_parser::core::account_event_parser::AccountEventParser; -use crate::streaming::event_parser::core::common_event_parser::CommonEventParser; -use crate::streaming::event_parser::EventParser; -use crate::streaming::event_parser::{ - core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, -}; - -/// 事件处理器 -pub struct EventProcessor { - pub(crate) metrics_manager: MetricsManager, - pub(crate) config: ClientConfig, - pub(crate) parser_cache: Arc>>>, -} - -impl EventProcessor { - /// 创建新的事件处理器 - pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { - Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) } - } - - /// 获取或创建解析器，使用缓存机制避免重复创建 - fn get_or_create_parser( - &self, - protocols: Vec, - event_type_filter: Option, - ) -> Arc { - let mut cache = self.parser_cache.lock().unwrap(); - if let Some(cached_parser) = cache.clone() { - return cached_parser.clone(); - } - let parser: Arc = - Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone())); - *cache = Some(parser.clone()); - parser - } - - /// 使用性能监控处理事件交易 - pub async fn process_event_transaction_with_metrics( - &self, - event_pretty: EventPretty, - callback: &F, - bot_wallet: Option, - protocols: Vec, - event_type_filter: Option, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { - match event_pretty { - EventPretty::Account(account_pretty) => { - self.metrics_manager.add_account_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let account_event = AccountEventParser::parse_account_event( - protocols.clone(), - account_pretty, - program_received_time_ms, - event_type_filter, - ); - if let Some(event) = account_event { - callback(event); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 更新性能指标（如果启用） - self.metrics_manager - .update_metrics(MetricsEventType::Account, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - EventPretty::Transaction(transaction_pretty) => { - self.metrics_manager.add_tx_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_pretty.slot; - let signature = transaction_pretty.signature.to_string(); - - // 使用缓存获取解析器 - let parser = self.get_or_create_parser(protocols.clone(), event_type_filter); - let all_events = parser - .parse_transaction( - transaction_pretty.tx.clone(), - &signature, - Some(slot), - transaction_pretty.block_time.map(|ts| prost_types::Timestamp { - seconds: ts.seconds, - nanos: ts.nanos, - }), - program_received_time_ms, - bot_wallet, - ) - .await - .unwrap_or_else(|_e| vec![]); - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 批量处理事件 - if !all_events.is_empty() { - for event in all_events { - callback(event); - } - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 更新性能指标（如果启用） - self.metrics_manager - .update_metrics(MetricsEventType::Tx, event_count as u64, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, event_count); - } - EventPretty::BlockMeta(block_meta_pretty) => { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_block_meta_process_count().await; - let block_time_ms = block_meta_pretty - .block_time - .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000) - .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); - let block_meta_event = CommonEventParser::generate_block_meta_event( - block_meta_pretty.slot, - &block_meta_pretty.block_hash, - block_time_ms, - ); - callback(block_meta_event); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 更新性能指标（如果启用） - self.metrics_manager - .update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - - Ok(()) - } - - /// 使用批处理处理事件交易 - pub async fn process_event_transaction_with_batch( - &self, - event_pretty: EventPretty, - batch_processor: &mut EventBatchCollector, - bot_wallet: Option, - protocols: Vec, - event_type_filter: Option, - ) -> AnyResult<()> - where - F: Fn(Vec>) + Send + Sync + 'static, - { - match event_pretty { - EventPretty::Account(account_pretty) => { - self.metrics_manager.add_account_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let account_event = AccountEventParser::parse_account_event( - protocols.clone(), - account_pretty, - program_received_time_ms, - event_type_filter, - ); - if let Some(event) = account_event { - (batch_processor.callback)(vec![event]); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 实际调用性能指标更新 - self.metrics_manager - .update_metrics(MetricsEventType::Account, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - EventPretty::Transaction(transaction_pretty) => { - self.metrics_manager.add_tx_process_count().await; - let start_time = std::time::Instant::now(); - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_pretty.slot; - let signature = transaction_pretty.signature.to_string(); - - // 使用缓存获取解析器 - let parser = self.get_or_create_parser(protocols.clone(), event_type_filter); - let result = parser - .parse_transaction( - transaction_pretty.tx.clone(), - &signature, - Some(slot), - transaction_pretty.block_time.map(|ts| prost_types::Timestamp { - seconds: ts.seconds, - nanos: ts.nanos, - }), - program_received_time_ms, - bot_wallet, - ) - .await; - - // 处理解析结果并使用批处理器 - let total_events = match result { - Ok(events) => { - let event_count = events.len(); - if !events.is_empty() { - log::debug!("Parsed {} events", event_count); - log::debug!("Adding {} events to batch processor", event_count); - for event in events { - if self.config.batch.enabled { - batch_processor.add_event(event); - } else { - // 如果批处理被禁用，直接调用回调 - // 这里需要将单个事件包装成Vec来调用批处理回调 - let single_event_batch = vec![event]; - (batch_processor.callback)(single_event_batch); - } - } - } - event_count - } - Err(e) => { - log::warn!("Failed to parse transaction: {:?}", e); - 0 - } - }; - - // 添加调试信息 - if total_events > 0 { - log::debug!( - "Total events parsed: {} for transaction {}", - total_events, - signature - ); - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 实际调用性能指标更新 - self.metrics_manager - .update_metrics(MetricsEventType::Tx, total_events as u64, processing_time_ms) - .await; - - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, total_events); - } - EventPretty::BlockMeta(block_meta_pretty) => { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_block_meta_process_count().await; - let block_time_ms = block_meta_pretty - .block_time - .map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000) - .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); - let block_meta_event = CommonEventParser::generate_block_meta_event( - block_meta_pretty.slot, - &block_meta_pretty.block_hash, - block_time_ms, - ); - (batch_processor.callback)(vec![block_meta_event]); - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - // 更新性能指标（如果启用） - self.metrics_manager - .update_metrics(MetricsEventType::BlockMeta, 1, processing_time_ms) - .await; - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, 1); - } - } - - Ok(()) - } -} diff --git a/src/streaming/grpc/mod.rs b/src/streaming/grpc/mod.rs index 44713a9..9c012e6 100644 --- a/src/streaming/grpc/mod.rs +++ b/src/streaming/grpc/mod.rs @@ -1,25 +1,17 @@ // gRPC 相关模块 pub mod connection; -pub mod types; -pub mod subscription; pub mod stream_handler; -pub mod event_processor; +pub mod subscription; +pub mod types; // 重新导出主要类型 pub use connection::*; -pub use types::*; -pub use subscription::*; pub use stream_handler::*; -pub use event_processor::*; +pub use subscription::*; +pub use types::*; // 从公用模块重新导出 pub use crate::streaming::common::{ - StreamClientConfig as ClientConfig, - PerformanceMetrics, - MetricsManager, - EventBatchProcessor as EventBatchCollector, - BackpressureStrategy, - BatchConfig, - BackpressureConfig, - ConnectionConfig, -}; \ No newline at end of file + BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, MetricsManager, + PerformanceMetrics, StreamClientConfig as ClientConfig, +}; diff --git a/src/streaming/grpc/stream_handler.rs b/src/streaming/grpc/stream_handler.rs index 362d963..ff6fb00 100644 --- a/src/streaming/grpc/stream_handler.rs +++ b/src/streaming/grpc/stream_handler.rs @@ -1,12 +1,14 @@ use chrono::Local; use futures::{channel::mpsc, sink::Sink, SinkExt}; +use solana_sdk::pubkey::Pubkey; use yellowstone_grpc_proto::geyser::{ subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate, }; use super::types::{BlockMetaPretty, EventPretty, TransactionPretty}; use crate::common::AnyResult; -use crate::streaming::common::BackpressureStrategy; +use crate::streaming::common::EventProcessor; +use crate::streaming::event_parser::UnifiedEvent; use crate::streaming::grpc::AccountPretty; /// 流消息处理器 @@ -14,33 +16,39 @@ pub struct StreamHandler; impl StreamHandler { /// 处理单个流消息 - pub async fn handle_stream_message( + pub async fn handle_stream_message( msg: SubscribeUpdate, - tx: &mut mpsc::Sender, subscribe_tx: &mut (impl Sink + Unpin), - backpressure_strategy: BackpressureStrategy, - ) -> AnyResult<()> { + event_processor: EventProcessor, + callback: &F, + bot_wallet: Option, + ) -> AnyResult<()> + where + F: Fn(Box) + Send + Sync, + { let created_at = msg.created_at; match msg.update_oneof { Some(UpdateOneof::Account(account)) => { let account_pretty = AccountPretty::from(account); log::debug!("Received account: {:?}", account_pretty); - Self::handle_backpressure( - tx, - EventPretty::Account(account_pretty), - backpressure_strategy, - ) - .await?; + event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::Account(account_pretty), + callback, + bot_wallet, + ) + .await?; } Some(UpdateOneof::BlockMeta(sut)) => { let block_meta_pretty = BlockMetaPretty::from((sut, created_at)); log::debug!("Received block meta: {:?}", block_meta_pretty); - Self::handle_backpressure( - tx, - EventPretty::BlockMeta(block_meta_pretty), - backpressure_strategy, - ) - .await?; + event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::BlockMeta(block_meta_pretty), + callback, + bot_wallet, + ) + .await?; } Some(UpdateOneof::Transaction(sut)) => { let transaction_pretty = TransactionPretty::from((sut, created_at)); @@ -49,14 +57,13 @@ impl StreamHandler { transaction_pretty.signature, transaction_pretty.slot ); - - // 根据背压策略处理发送 - Self::handle_backpressure( - tx, - EventPretty::Transaction(transaction_pretty), - backpressure_strategy, - ) - .await?; + event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::Transaction(transaction_pretty), + callback, + bot_wallet, + ) + .await?; } Some(UpdateOneof::Ping(_)) => { subscribe_tx @@ -77,58 +84,58 @@ impl StreamHandler { Ok(()) } - /// 处理背压策略 - async fn handle_backpressure( - tx: &mut mpsc::Sender, - event_pretty: EventPretty, - backpressure_strategy: BackpressureStrategy, - ) -> AnyResult<()> { - match backpressure_strategy { - BackpressureStrategy::Block => { - // 阻塞等待，直到有空间 - if let Err(e) = tx.send(event_pretty).await { - log::error!("Failed to send transaction to channel: {:?}", e); - return Err(anyhow::anyhow!("Channel send failed: {:?}", e)); - } - } - BackpressureStrategy::Drop => { - // 尝试发送，如果失败则丢弃 - if let Err(e) = tx.try_send(event_pretty) { - if e.is_full() { - log::warn!("Channel is full, dropping transaction"); - } else { - log::error!("Channel is closed: {:?}", e); - return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); - } - } - } - BackpressureStrategy::Retry { max_attempts, wait_ms } => { - // 重试有限次数 - let mut retry_count = 0; - loop { - match tx.try_send(event_pretty.clone()) { - Ok(_) => break, - Err(e) => { - if e.is_full() { - retry_count += 1; - if retry_count >= max_attempts { - log::warn!( - "Channel is full after {} attempts, dropping transaction", - retry_count - ); - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms)) - .await; - } else { - log::error!("Channel is closed: {:?}", e); - return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); - } - } - } - } - } - } - Ok(()) - } + // /// 处理背压策略 + // async fn handle_backpressure( + // tx: &mut mpsc::Sender, + // event_pretty: EventPretty, + // backpressure_strategy: BackpressureStrategy, + // ) -> AnyResult<()> { + // match backpressure_strategy { + // BackpressureStrategy::Block => { + // // 阻塞等待，直到有空间 + // if let Err(e) = tx.send(event_pretty).await { + // log::error!("Failed to send transaction to channel: {:?}", e); + // return Err(anyhow::anyhow!("Channel send failed: {:?}", e)); + // } + // } + // BackpressureStrategy::Drop => { + // // 尝试发送，如果失败则丢弃 + // if let Err(e) = tx.try_send(event_pretty) { + // if e.is_full() { + // log::warn!("Channel is full, dropping transaction"); + // } else { + // log::error!("Channel is closed: {:?}", e); + // return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); + // } + // } + // } + // BackpressureStrategy::Retry { max_attempts, wait_ms } => { + // // 重试有限次数 + // let mut retry_count = 0; + // loop { + // match tx.try_send(event_pretty.clone()) { + // Ok(_) => break, + // Err(e) => { + // if e.is_full() { + // retry_count += 1; + // if retry_count >= max_attempts { + // log::warn!( + // "Channel is full after {} attempts, dropping transaction", + // retry_count + // ); + // break; + // } + // tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms)) + // .await; + // } else { + // log::error!("Channel is closed: {:?}", e); + // return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); + // } + // } + // } + // } + // } + // } + // Ok(()) + // } } diff --git a/src/streaming/grpc/types.rs b/src/streaming/grpc/types.rs index 8d80a7c..b0ef05f 100644 --- a/src/streaming/grpc/types.rs +++ b/src/streaming/grpc/types.rs @@ -1,5 +1,5 @@ -use solana_sdk::signature::Signature; -use solana_transaction_status::{EncodedTransactionWithStatusMeta, UiTransactionEncoding}; +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use solana_transaction_status::TransactionWithStatusMeta; use std::{collections::HashMap, fmt}; use yellowstone_grpc_proto::{ geyser::{ @@ -22,13 +22,14 @@ pub enum EventPretty { #[derive(Clone)] pub struct AccountPretty { pub slot: u64, - pub signature: String, - pub pubkey: String, + pub signature: Signature, + pub pubkey: Pubkey, pub executable: bool, pub lamports: u64, - pub owner: String, + pub owner: Pubkey, pub rent_epoch: u64, pub data: Vec, + pub program_received_time_us: i64, } impl fmt::Debug for AccountPretty { @@ -51,6 +52,7 @@ pub struct BlockMetaPretty { pub slot: u64, pub block_hash: String, pub block_time: Option, + pub program_received_time_us: i64, } impl fmt::Debug for BlockMetaPretty { @@ -59,6 +61,7 @@ impl fmt::Debug for BlockMetaPretty { .field("slot", &self.slot) .field("block_hash", &self.block_hash) .field("block_time", &self.block_time) + .field("program_received_time_us", &self.program_received_time_us) .finish() } } @@ -70,24 +73,17 @@ pub struct TransactionPretty { pub block_time: Option, pub signature: Signature, pub is_vote: bool, - pub tx: EncodedTransactionWithStatusMeta, + pub tx: TransactionWithStatusMeta, + pub program_received_time_us: i64, } impl fmt::Debug for TransactionPretty { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - struct TxWrap<'a>(&'a EncodedTransactionWithStatusMeta); - impl<'a> fmt::Debug for TxWrap<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let serialized = serde_json::to_string(self.0).expect("failed to serialize"); - fmt::Display::fmt(&serialized, f) - } - } - f.debug_struct("TransactionPretty") .field("slot", &self.slot) .field("signature", &self.signature) .field("is_vote", &self.is_vote) - .field("tx", &TxWrap(&self.tx)) + .field("program_received_time_us", &self.program_received_time_us) .finish() } } @@ -97,13 +93,17 @@ impl From for AccountPretty { let account_info = account.account.unwrap(); Self { slot: account.slot, - signature: bs58::encode(&account_info.txn_signature.unwrap_or_default()).into_string(), - pubkey: bs58::encode(&account_info.pubkey).into_string(), + signature: Signature::try_from( + account_info.txn_signature.unwrap_or_default().as_slice(), + ) + .expect("valid signature"), + pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"), executable: account_info.executable, lamports: account_info.lamports, - owner: bs58::encode(&account_info.owner).into_string(), + owner: Pubkey::try_from(account_info.owner.as_slice()).expect("valid pubkey"), rent_epoch: account_info.rent_epoch, data: account_info.data, + program_received_time_us: chrono::Utc::now().timestamp_micros(), } } } @@ -115,7 +115,12 @@ impl From<(SubscribeUpdateBlockMeta, Option)> for BlockMetaPretty { Option, ), ) -> Self { - Self { block_hash: blockhash.to_string(), block_time, slot } + Self { + block_hash: blockhash.to_string(), + block_time, + slot, + program_received_time_us: chrono::Utc::now().timestamp_micros(), + } } } @@ -134,9 +139,8 @@ impl From<(SubscribeUpdateTransaction, Option)> for TransactionPretty signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"), is_vote: tx.is_vote, tx: yellowstone_grpc_proto::convert_from::create_tx_with_meta(tx) - .expect("valid tx with meta") - .encode(UiTransactionEncoding::Base64, Some(u8::MAX), true) - .expect("failed to encode"), + .expect("valid tx with meta"), + program_received_time_us: chrono::Utc::now().timestamp_micros(), } } } diff --git a/src/streaming/mod.rs b/src/streaming/mod.rs index 8747e2a..9aed6b5 100755 --- a/src/streaming/mod.rs +++ b/src/streaming/mod.rs @@ -4,8 +4,8 @@ pub mod grpc; pub mod shred; pub mod shred_stream; pub mod yellowstone_grpc; -pub mod yellowstone_sub_system; +// pub mod yellowstone_sub_system; pub use shred::ShredStreamGrpc; pub use yellowstone_grpc::YellowstoneGrpc; -pub use yellowstone_sub_system::{SystemEvent, TransferInfo}; +// pub use yellowstone_sub_system::{SystemEvent, TransferInfo}; diff --git a/src/streaming/shred/connection.rs b/src/streaming/shred/connection.rs index 8a27a8a..90bc64a 100644 --- a/src/streaming/shred/connection.rs +++ b/src/streaming/shred/connection.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::RwLock; use tokio::sync::Mutex; use tonic::transport::Channel; @@ -13,7 +14,7 @@ use crate::streaming::common::{ pub struct ShredStreamGrpc { pub shredstream_client: Arc>, pub config: StreamClientConfig, - pub metrics: Arc>, + pub metrics: Arc>, pub metrics_manager: MetricsManager, pub subscription_handle: Arc>>, } @@ -27,7 +28,7 @@ impl ShredStreamGrpc { /// 创建客户端，使用自定义配置 pub async fn new_with_config(endpoint: String, config: StreamClientConfig) -> AnyResult { let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?; - let metrics = Arc::new(Mutex::new(PerformanceMetrics::new())); + let metrics = Arc::new(RwLock::new(PerformanceMetrics::new())); let config_arc = Arc::new(config.clone()); let metrics_manager = @@ -36,7 +37,7 @@ impl ShredStreamGrpc { Ok(Self { shredstream_client: Arc::new(shredstream_client), config, - metrics, + metrics: metrics.clone(), metrics_manager, subscription_handle: Arc::new(Mutex::new(None)), }) @@ -63,8 +64,8 @@ impl ShredStreamGrpc { } /// 获取性能指标 - pub async fn get_metrics(&self) -> PerformanceMetrics { - self.metrics_manager.get_metrics().await + pub fn get_metrics(&self) -> PerformanceMetrics { + self.metrics_manager.get_metrics() } /// 启用或禁用性能监控 @@ -73,8 +74,8 @@ impl ShredStreamGrpc { } /// 打印性能指标 - pub async fn print_metrics(&self) { - self.metrics_manager.print_metrics().await; + pub fn print_metrics(&self) { + self.metrics_manager.print_metrics(); } /// 启动自动性能监控任务 diff --git a/src/streaming/shred/event_processor.rs b/src/streaming/shred/event_processor.rs deleted file mode 100644 index c72895a..0000000 --- a/src/streaming/shred/event_processor.rs +++ /dev/null @@ -1,170 +0,0 @@ -use solana_sdk::pubkey::Pubkey; -use std::sync::{Arc, Mutex}; - -use crate::common::AnyResult; -use crate::streaming::common::{ - EventBatchProcessor, MetricsEventType, MetricsManager, StreamClientConfig, -}; -use crate::streaming::event_parser::common::filter::EventTypeFilter; -use crate::streaming::event_parser::protocols::MutilEventParser; -use crate::streaming::event_parser::{EventParser, Protocol, UnifiedEvent}; -use crate::streaming::shred::TransactionWithSlot; - -/// ShredStream 事件处理器 -pub struct ShredEventProcessor { - pub(crate) metrics_manager: MetricsManager, - pub(crate) config: StreamClientConfig, - pub(crate) parser_cache: Arc>>>, -} - -impl ShredEventProcessor { - /// 创建新的事件处理器 - pub fn new(metrics_manager: MetricsManager, config: StreamClientConfig) -> Self { - Self { metrics_manager, config, parser_cache: Arc::new(Mutex::new(None)) } - } - - /// 获取或创建解析器，使用缓存机制避免重复创建 - fn get_or_create_parser( - &self, - protocols: Vec, - event_type_filter: Option, - ) -> Arc { - let mut cache = self.parser_cache.lock().unwrap(); - if let Some(cached_parser) = cache.clone() { - return cached_parser.clone(); - } - let parser: Arc = - Arc::new(MutilEventParser::new(protocols.clone(), event_type_filter.clone())); - *cache = Some(parser.clone()); - parser - } - - /// 即时处理单个交易 - pub async fn process_transaction_immediate( - &self, - transaction_with_slot: TransactionWithSlot, - protocols: Vec, - bot_wallet: Option, - event_type_filter: Option, - callback: &F, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_tx_process_count().await; - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_with_slot.slot; - let versioned_tx = transaction_with_slot.transaction; - let signature = versioned_tx.signatures[0]; - - // 获取缓存的解析器 - let parser = self.get_or_create_parser(protocols, event_type_filter); - - let all_events = parser - .parse_versioned_transaction( - &versioned_tx, - &signature.to_string(), - Some(slot), - None, - program_received_time_ms, - bot_wallet, - ) - .await - .unwrap_or_else(|_e| vec![]); - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 即时处理事件 - for event in all_events { - callback(event); - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 实际调用性能指标更新 - self.update_metrics(event_count as u64, processing_time_ms).await; - - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, event_count); - - Ok(()) - } - - /// 批处理模式处理单个交易 - pub async fn process_transaction_with_batch( - &self, - transaction_with_slot: TransactionWithSlot, - protocols: Vec, - bot_wallet: Option, - batch_processor: &mut EventBatchProcessor, - event_type_filter: Option, - ) -> AnyResult<()> - where - F: FnMut(Vec>) + Send + Sync + 'static, - { - let start_time = std::time::Instant::now(); - self.metrics_manager.add_tx_process_count().await; - let program_received_time_ms = chrono::Utc::now().timestamp_millis(); - let slot = transaction_with_slot.slot; - let versioned_tx = transaction_with_slot.transaction; - let signature = versioned_tx.signatures[0]; - - // 获取缓存的解析器 - let parser = self.get_or_create_parser(protocols, event_type_filter); - - let all_events = parser - .parse_versioned_transaction( - &versioned_tx, - &signature.to_string(), - Some(slot), - None, - program_received_time_ms, - bot_wallet, - ) - .await - .unwrap_or_else(|_e| vec![]); - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 使用批处理器处理事件 - for event in all_events { - batch_processor.add_event(event); - } - - // 更新性能指标 - let processing_time = start_time.elapsed(); - let processing_time_ms = processing_time.as_millis() as f64; - - // 实际调用性能指标更新 - self.update_metrics(event_count as u64, processing_time_ms).await; - - // 记录慢处理操作 - self.metrics_manager.log_slow_processing(processing_time_ms, event_count); - - Ok(()) - } - - /// 更新性能指标 - async fn update_metrics(&self, events_processed: u64, processing_time_ms: f64) { - // 使用统一的指标管理器，这里假设 ShredStream 主要处理交易事件 - self.metrics_manager - .update_metrics(MetricsEventType::Tx, events_processed, processing_time_ms) - .await; - } -} - -// 实现 Clone trait 以支持模块间共享 -impl Clone for ShredEventProcessor { - fn clone(&self) -> Self { - Self { - metrics_manager: self.metrics_manager.clone(), - config: self.config.clone(), - parser_cache: self.parser_cache.clone(), - } - } -} diff --git a/src/streaming/shred/mod.rs b/src/streaming/shred/mod.rs index 5ba7daf..c2aa2d9 100644 --- a/src/streaming/shred/mod.rs +++ b/src/streaming/shred/mod.rs @@ -1,17 +1,13 @@ // ShredStream 相关模块 pub mod connection; pub mod types; -pub mod stream_handler; -pub mod event_processor; // 重新导出主要类型 pub use connection::*; pub use types::*; -pub use stream_handler::*; -pub use event_processor::*; // 从公用模块重新导出 pub use crate::streaming::common::{ - BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, EventBatchProcessor, - MetricsEventType, MetricsManager, PerformanceMetrics, StreamClientConfig, + BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, MetricsEventType, + MetricsManager, PerformanceMetrics, StreamClientConfig, }; diff --git a/src/streaming/shred/stream_handler.rs b/src/streaming/shred/stream_handler.rs deleted file mode 100644 index b5de4c2..0000000 --- a/src/streaming/shred/stream_handler.rs +++ /dev/null @@ -1,116 +0,0 @@ -use futures::{channel::mpsc, StreamExt}; -use log::error; -use solana_entry::entry::Entry; -use tokio::task::JoinHandle; - -use crate::common::AnyResult; -use crate::protos::shredstream::{ - shredstream_proxy_client::ShredstreamProxyClient, SubscribeEntriesRequest, -}; -use crate::streaming::shred::TransactionWithSlot; - -/// ShredStream 流处理器 -pub struct ShredStreamHandler; - -impl ShredStreamHandler { - /// 启动 ShredStream 流处理任务 - /// - /// # 参数 - /// * `client` - ShredStream 客户端 - /// * `tx` - 事务发送通道 - /// * `channel_size` - 通道缓冲区大小 - /// - /// # 返回值 - /// 返回 ShredStream 流处理任务句柄和事务接收通道 - pub async fn start_stream_processing( - mut client: ShredstreamProxyClient, - channel_size: usize, - ) -> AnyResult<(JoinHandle<()>, mpsc::Receiver)> { - let request = tonic::Request::new(SubscribeEntriesRequest {}); - let stream = client.subscribe_entries(request).await?.into_inner(); - let (tx, rx) = mpsc::channel::(channel_size); - - let stream_task = tokio::spawn(Self::process_stream_messages(stream, tx)); - - Ok((stream_task, rx)) - } - - /// 处理流消息 - /// - /// # 参数 - /// * `stream` - ShredStream 数据流 - /// * `tx` - 事务发送通道 - async fn process_stream_messages( - mut stream: tonic::codec::Streaming, - mut tx: mpsc::Sender, - ) { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = Self::handle_stream_message(msg, &mut tx).await { - error!("Error handling stream message: {e:?}"); - continue; - } - } - Err(error) => { - error!("Stream error: {error:?}"); - break; - } - } - } - } - - /// 处理单个流消息 - /// - /// # 参数 - /// * `msg` - ShredStream 消息 - /// * `tx` - 事务发送通道 - async fn handle_stream_message( - msg: crate::protos::shredstream::Entry, - tx: &mut mpsc::Sender, - ) -> AnyResult<()> { - if let Ok(entries) = bincode::deserialize::>(&msg.entries) { - for entry in entries { - for transaction in entry.transactions { - let transaction_with_slot = - TransactionWithSlot::new(transaction.clone(), msg.slot); - - if let Err(e) = tx.try_send(transaction_with_slot) { - // 如果通道满了，记录警告但不中断处理 - if e.is_full() { - log::warn!("Transaction channel is full, dropping transaction"); - } else { - // 通道已关闭，返回错误 - return Err(e.into()); - } - } - } - } - } - Ok(()) - } - - /// 启动事务处理任务 - /// - /// # 参数 - /// * `rx` - 事务接收通道 - /// * `processor` - 事务处理器 - pub fn start_transaction_processing( - mut rx: mpsc::Receiver, - processor: F, - ) -> JoinHandle<()> - where - F: Fn(TransactionWithSlot) -> Result<(), Box> - + Send - + Sync - + 'static, - { - tokio::spawn(async move { - while let Some(transaction_with_slot) = rx.next().await { - if let Err(e) = processor(transaction_with_slot) { - error!("Error processing transaction: {e:?}"); - } - } - }) - } -} diff --git a/src/streaming/shred_stream.rs b/src/streaming/shred_stream.rs index 356e774..860429a 100755 --- a/src/streaming/shred_stream.rs +++ b/src/streaming/shred_stream.rs @@ -1,10 +1,14 @@ +use futures::StreamExt; use solana_sdk::pubkey::Pubkey; use crate::common::AnyResult; -use crate::streaming::common::{EventBatchProcessor, SubscriptionHandle}; +use crate::protos::shredstream::SubscribeEntriesRequest; +use crate::streaming::common::{EventProcessor, SubscriptionHandle}; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::{Protocol, UnifiedEvent}; -use crate::streaming::shred::{ShredEventProcessor, ShredStreamHandler, TransactionWithSlot}; +use crate::streaming::shred::TransactionWithSlot; +use log::error; +use solana_entry::entry::Entry; use super::ShredStreamGrpc; @@ -29,120 +33,63 @@ impl ShredStreamGrpc { metrics_handle = self.metrics_manager.start_auto_monitoring().await; } - // 启动流处理 - let client = (*self.shredstream_client).clone(); - let (stream_task, rx) = ShredStreamHandler::start_stream_processing( - client, - self.config.backpressure.channel_size, - ) - .await?; + // 创建事件处理器 + let mut event_processor = + EventProcessor::new(self.metrics_manager.clone(), self.config.clone()); + event_processor.set_protocols_and_event_type_filter( + protocols, + event_type_filter, + self.config.backpressure.strategy, + self.config.batch.clone(), + ); - // 根据配置选择处理模式并获取事件处理任务句柄 - let event_handle = if self.config.batch.enabled { - // 批处理模式 - self.process_with_batch(rx, protocols, bot_wallet, event_type_filter, callback).await? - } else { - // 即时处理模式 - self.process_immediate(rx, protocols, bot_wallet, event_type_filter, callback).await? - }; + // 启动流处理 + let mut client = (*self.shredstream_client).clone(); + let request = tonic::Request::new(SubscribeEntriesRequest {}); + let mut stream = client.subscribe_entries(request).await?.into_inner(); + let event_processor_clone = event_processor.clone(); + let stream_task = tokio::spawn(async move { + while let Some(message) = stream.next().await { + match message { + Ok(msg) => { + if let Ok(entries) = bincode::deserialize::>(&msg.entries) { + for entry in entries { + for transaction in entry.transactions { + let transaction_with_slot = + TransactionWithSlot::new(transaction.clone(), msg.slot); + if let Err(e) = event_processor_clone + .process_shred_transaction_immediate( + transaction_with_slot, + bot_wallet, + &callback, + ) + .await + { + error!("Error handling message: {e:?}"); + break; + } + } + } + } + continue; + } + Err(error) => { + error!("Stream error: {error:?}"); + break; + } + } + } + }); // 保存订阅句柄 - let subscription_handle = SubscriptionHandle::new(stream_task, event_handle, metrics_handle); + let subscription_handle = SubscriptionHandle::new( + stream_task, + event_processor.get_event_handle(), + metrics_handle, + ); let mut handle_guard = self.subscription_handle.lock().await; *handle_guard = Some(subscription_handle); Ok(()) } - - /// 批处理模式 - async fn process_with_batch( - &self, - mut rx: futures::channel::mpsc::Receiver, - protocols: Vec, - bot_wallet: Option, - event_type_filter: Option, - callback: F, - ) -> AnyResult> - where - F: Fn(Box) + Send + Sync + 'static, - { - use futures::StreamExt; - - // 创建批处理器，将单个事件回调转换为批量回调 - let batch_callback = move |events: Vec>| { - for event in events { - callback(event); - } - }; - - let mut batch_processor = EventBatchProcessor::new( - batch_callback, - self.config.batch.batch_size, - self.config.batch.batch_timeout_ms, - ); - - // 创建事件处理器 - let event_processor = - ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone()); - - let event_handle = tokio::spawn(async move { - while let Some(transaction_with_slot) = rx.next().await { - if let Err(e) = event_processor - .process_transaction_with_batch( - transaction_with_slot, - protocols.clone(), - bot_wallet, - &mut batch_processor, - event_type_filter.clone(), - ) - .await - { - log::error!("Error processing transaction: {e:?}"); - } - } - - // 处理剩余的事件 - batch_processor.flush(); - }); - - Ok(event_handle) - } - - /// 即时处理模式 - async fn process_immediate( - &self, - mut rx: futures::channel::mpsc::Receiver, - protocols: Vec, - bot_wallet: Option, - event_type_filter: Option, - callback: F, - ) -> AnyResult> - where - F: Fn(Box) + Send + Sync + 'static, - { - use futures::StreamExt; - - // 创建事件处理器 - let event_processor = - ShredEventProcessor::new(self.metrics_manager.clone(), self.config.clone()); - - let event_handle = tokio::spawn(async move { - while let Some(transaction_with_slot) = rx.next().await { - if let Err(e) = event_processor - .process_transaction_immediate( - transaction_with_slot, - protocols.clone(), - bot_wallet, - event_type_filter.clone(), - &callback, - ) - .await - { - log::error!("Error processing transaction: {e:?}"); - } - } - }); - - Ok(event_handle) - } } diff --git a/src/streaming/yellowstone_grpc.rs b/src/streaming/yellowstone_grpc.rs index f88e88b..58075cd 100644 --- a/src/streaming/yellowstone_grpc.rs +++ b/src/streaming/yellowstone_grpc.rs @@ -1,17 +1,17 @@ -use futures::{channel::mpsc, StreamExt}; +use futures::StreamExt; use log::error; use solana_sdk::pubkey::Pubkey; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use tokio::sync::Mutex; use yellowstone_grpc_proto::geyser::CommitmentLevel; use crate::common::AnyResult; use crate::streaming::common::{ - EventBatchProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle, + EventProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle, }; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::{Protocol, UnifiedEvent}; -use crate::streaming::grpc::{EventPretty, EventProcessor, StreamHandler, SubscriptionManager}; +use crate::streaming::grpc::{StreamHandler, SubscriptionManager}; /// 交易过滤器 pub struct TransactionFilter { @@ -30,7 +30,7 @@ pub struct YellowstoneGrpc { pub endpoint: String, pub x_token: Option, pub config: StreamClientConfig, - pub metrics: Arc>, + pub metrics: Arc>, pub subscription_manager: SubscriptionManager, pub metrics_manager: MetricsManager, pub event_processor: EventProcessor, @@ -50,7 +50,7 @@ impl YellowstoneGrpc { config: StreamClientConfig, ) -> AnyResult { let _ = rustls::crypto::ring::default_provider().install_default().ok(); - let metrics = Arc::new(Mutex::new(PerformanceMetrics::new())); + let metrics = Arc::new(RwLock::new(PerformanceMetrics::new())); let config_arc = Arc::new(config.clone()); let subscription_manager = @@ -63,7 +63,7 @@ impl YellowstoneGrpc { endpoint, x_token, config, - metrics, + metrics: metrics.clone(), subscription_manager, metrics_manager, event_processor, @@ -99,13 +99,13 @@ impl YellowstoneGrpc { } /// 获取性能指标 - pub async fn get_metrics(&self) -> PerformanceMetrics { - self.metrics_manager.get_metrics().await + pub fn get_metrics(&self) -> PerformanceMetrics { + self.metrics_manager.get_metrics() } /// 打印性能指标 - pub async fn print_metrics(&self) { - self.metrics_manager.print_metrics().await; + pub fn print_metrics(&self) { + self.metrics_manager.print_metrics(); } /// 启用或禁用性能监控 @@ -174,20 +174,24 @@ impl YellowstoneGrpc { .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) .await?; - // 创建通道，使用配置中的通道大小 - let (mut tx, mut rx) = mpsc::channel::(self.config.backpressure.channel_size); - // 启动流处理任务 - let backpressure_strategy = self.config.backpressure.strategy; + let mut event_processor = self.event_processor.clone(); + event_processor.set_protocols_and_event_type_filter( + protocols, + event_type_filter, + self.config.backpressure.strategy, + self.config.batch.clone(), + ); let stream_handle = tokio::spawn(async move { while let Some(message) = stream.next().await { match message { Ok(msg) => { if let Err(e) = StreamHandler::handle_stream_message( msg, - &mut tx, &mut subscribe_tx, - backpressure_strategy, + event_processor.clone(), + &callback, + bot_wallet, ) .await { @@ -203,160 +207,12 @@ impl YellowstoneGrpc { } }); - // 即时处理交易，无批处理 - let event_processor = self.event_processor.clone(); - let event_handle = tokio::spawn(async move { - while let Some(event_pretty) = rx.next().await { - if let Err(e) = event_processor - .process_event_transaction_with_metrics( - event_pretty, - &callback, - bot_wallet, - protocols.clone(), - event_type_filter.clone(), - ) - .await - { - error!("Error processing transaction: {e:?}"); - } - } - }); - // 保存订阅句柄 - let subscription_handle = - SubscriptionHandle::new(stream_handle, event_handle, metrics_handle); - let mut handle_guard = self.subscription_handle.lock().await; - *handle_guard = Some(subscription_handle); - - Ok(()) - } - - /// Advanced event subscription with batch processing and backpressure handling - /// - /// # Parameters - /// * `protocols` - List of protocols to monitor - /// * `bot_wallet` - Optional bot wallet address for filtering related transactions - /// * `transaction_filter` - Transaction filter specifying accounts to include/exclude - /// * `account_filter` - Account filter specifying accounts and owners to monitor - /// * `event_filter` - Optional event filter for further event filtering, no filtering if None - /// * `commitment` - Optional commitment level, defaults to Confirmed - /// * `callback` - Event callback function that receives parsed unified events - /// - /// # Features - /// * Batch processing for improved throughput - /// * Backpressure handling to prevent memory overflow - /// * Automatic performance monitoring (if enabled) - /// * Configurable batch size and timeout - /// - /// # Returns - /// Returns `AnyResult<()>`, `Ok(())` on success, error information on failure - pub async fn subscribe_events_advanced( - &self, - protocols: Vec, - bot_wallet: Option, - transaction_filter: TransactionFilter, - account_filter: AccountFilter, - event_type_filter: Option, - commitment: Option, - callback: F, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync + 'static, - { - // 如果已有活跃订阅，先停止它 - self.stop().await; - - let mut metrics_handle = None; - // 启动自动性能监控（如果启用） - if self.config.enable_metrics { - metrics_handle = self.metrics_manager.start_auto_monitoring().await; - } - - let transactions = self.subscription_manager.get_subscribe_request_filter( - transaction_filter.account_include, - transaction_filter.account_exclude, - transaction_filter.account_required, - event_type_filter.clone(), + let subscription_handle = SubscriptionHandle::new( + stream_handle, + self.event_processor.get_event_handle(), + metrics_handle, ); - let accounts = self.subscription_manager.subscribe_with_account_request( - account_filter.account, - account_filter.owner, - event_type_filter.clone(), - ); - - // Subscribe to events - let (mut subscribe_tx, mut stream) = self - .subscription_manager - .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) - .await?; - - // Create channel - let (mut tx, mut rx) = mpsc::channel::(self.config.backpressure.channel_size); - - // 创建批处理器，将单个事件回调转换为批量回调 - let batch_callback = move |events: Vec>| { - for event in events { - callback(event); - } - }; - - let mut batch_processor = EventBatchProcessor::new( - batch_callback, - self.config.batch.batch_size, - self.config.batch.batch_timeout_ms, - ); - - // Start task to process the stream - let backpressure_strategy = self.config.backpressure.strategy; - let stream_handle = tokio::spawn(async move { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - backpressure_strategy, - ) - .await - { - error!("Error handling message: {e:?}"); - break; - } - } - Err(error) => { - error!("Stream error: {error:?}"); - break; - } - } - } - }); - - // Process transactions with batch processing - let event_processor = self.event_processor.clone(); - let event_handle = tokio::spawn(async move { - while let Some(event_pretty) = rx.next().await { - if let Err(e) = event_processor - .process_event_transaction_with_batch( - event_pretty, - &mut batch_processor, - bot_wallet, - protocols.clone(), - event_type_filter.clone(), - ) - .await - { - error!("Error processing transaction: {e:?}"); - } - } - - // 处理剩余的事件 - batch_processor.flush(); - }); - - // 保存订阅句柄 - let subscription_handle = - SubscriptionHandle::new(stream_handle, event_handle, metrics_handle); let mut handle_guard = self.subscription_handle.lock().await; *handle_guard = Some(subscription_handle); @@ -379,14 +235,3 @@ impl Clone for YellowstoneGrpc { } } } - -// 实现 Clone trait 以支持模块间共享 -impl Clone for EventProcessor { - fn clone(&self) -> Self { - Self { - metrics_manager: self.metrics_manager.clone(), - config: self.config.clone(), - parser_cache: self.parser_cache.clone(), - } - } -} diff --git a/src/streaming/yellowstone_sub_system.rs b/src/streaming/yellowstone_sub_system.rs index ebb4aa9..4d19f95 100755 --- a/src/streaming/yellowstone_sub_system.rs +++ b/src/streaming/yellowstone_sub_system.rs @@ -9,7 +9,7 @@ use futures::{channel::mpsc, StreamExt}; use log::error; use solana_program::pubkey; use solana_sdk::{pubkey::Pubkey, transaction::VersionedTransaction}; -use solana_transaction_status::EncodedTransactionWithStatusMeta; +use solana_transaction_status::TransactionWithStatusMeta; const SYSTEM_PROGRAM_ID: Pubkey = pubkey!("11111111111111111111111111111111"); // 根据实际并发量调整通道大小，避免背压 @@ -93,20 +93,19 @@ impl YellowstoneGrpc { { match event_pretty { EventPretty::Transaction(transaction_pretty) => { - let trade_raw: EncodedTransactionWithStatusMeta = transaction_pretty.tx; - let meta = trade_raw - .meta - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?; + let trade_raw: TransactionWithStatusMeta = transaction_pretty.tx; + let meta = trade_raw.get_status_meta(); - if meta.err.is_some() { + if meta.is_none() { return Ok(()); } + let transaction = trade_raw.get_transaction(); + callback(SystemEvent::NewTransfer(TransferInfo { slot: transaction_pretty.slot, signature: transaction_pretty.signature.to_string(), - tx: trade_raw.transaction.decode(), + tx: Some(transaction), })); } _ => {} From 1e6841149d844a08b7f297356d651812d5d354c9 Mon Sep 17 00:00:00 2001 From: ioxde <228087182+ioxde@users.noreply.github.com> Date: Mon, 25 Aug 2025 17:09:14 -0700 Subject: [PATCH 3/7] feat: add dynamic subscription management with runtime filter updates - Add update_subscription() for runtime filter updates without reconnection - BREAKING: Return subscription request from subscribe_with_request() - Implement Default trait for EventTypeFilter - Add comprehensive dynamic_subscription example --- README.md | 41 ++ examples/dynamic_subscription.rs | 468 ++++++++++++++++++++ src/streaming/event_parser/common/filter.rs | 2 +- src/streaming/grpc/subscription.rs | 5 +- src/streaming/yellowstone_grpc.rs | 204 +++++++-- src/streaming/yellowstone_sub_system.rs | 2 +- 6 files changed, 677 insertions(+), 45 deletions(-) create mode 100644 examples/dynamic_subscription.rs diff --git a/README.md b/README.md index cc73cf0..db44d9c 100755 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ A lightweight Rust library for real-time event streaming from Solana DEX trading 16. **Runtime Configuration Updates**: Supports dynamic configuration parameter updates at runtime 17. **Full Function Performance Monitoring**: All subscribe_events functions support performance monitoring, automatically collecting and reporting performance metrics 18. **Graceful Shutdown**: Support for programmatic stop() method for clean shutdown +19. **Dynamic Subscription Management**: Runtime filter updates without reconnection, enabling adaptive monitoring strategies ## Installation @@ -72,6 +73,20 @@ This example demonstrates: The example uses a predefined transaction signature and shows how to extract protocol-specific events from the transaction data. +### Dynamic Subscription Management Example + +Test runtime filter updates without reconnection: + +```bash +cargo run --example dynamic_subscription +``` + +This example demonstrates: +- Creating initial subscriptions with specific protocol filters +- Updating subscription filters at runtime without reconnection +- Single subscription enforcement and proper error handling +- Clean shutdown and resource management + ### Advanced Usage - Complete Example ```rust @@ -470,6 +485,32 @@ let event_type_filter = Some(EventTypeFilter { }); ``` +## Dynamic Subscription Management + +Update subscription filters at runtime without reconnecting to the stream. + +```rust +// Update filters on existing subscription +grpc.update_subscription( + TransactionFilter { + account_include: vec!["new_program_id".to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, +).await?; +``` + +- **No Reconnection**: Filter changes apply immediately without closing the stream +- **Atomic Updates**: Both transaction and account filters updated together +- **Single Subscription**: One active subscription per client instance +- **Compatible**: Works with both immediate and advanced subscription methods + +Note: Multiple subscription attempts on the same client return an error. + ## Supported Protocols - **PumpFun**: Primary meme coin trading platform diff --git a/examples/dynamic_subscription.rs b/examples/dynamic_subscription.rs new file mode 100644 index 0000000..fcb7365 --- /dev/null +++ b/examples/dynamic_subscription.rs @@ -0,0 +1,468 @@ +use anyhow::Result; +use solana_streamer_sdk::streaming::yellowstone_grpc::{AccountFilter, TransactionFilter, YellowstoneGrpc}; +use solana_streamer_sdk::streaming::event_parser::Protocol; +use solana_streamer_sdk::streaming::event_parser::common::filter::EventTypeFilter; +use solana_streamer_sdk::streaming::event_parser::common::types::EventType; +use solana_sdk::signature::{Keypair, Signer}; +use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; +use tokio::time::sleep; + +const PUMPFUN_PROGRAM_ID: &str = "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P"; +const RAYDIUM_CPMM_PROGRAM_ID: &str = "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C"; + +const GRPC_ENDPOINT: &str = "https://solana-yellowstone-grpc.publicnode.com:443"; +const API_KEY: Option<&str> = None; +const MONITORING_DURATION_SECS: u64 = 2; + +/// Demonstrates dynamic subscription updates and filter changes in real-time +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + + println!("Connecting to Yellowstone gRPC at {}", GRPC_ENDPOINT); + let client = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let event_counter = Arc::new(AtomicU64::new(0)); + let counter = event_counter.clone(); + + let callback = move |event: Box| { + let count = counter.fetch_add(1, Ordering::Relaxed); + + let protocol = match event.event_type() { + EventType::PumpFunBuy | EventType::PumpFunSell => "PumpFun", + EventType::RaydiumCpmmSwapBaseInput | EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmm", + _ => "Unknown" + }; + + println!("Event #{}: {:11} - {:.8}...", count + 1, protocol, event.signature()); + }; + + println!("\n=== Phase 1: PumpFun only ==="); + let pumpfun_filter = TransactionFilter { + account_include: vec![PUMPFUN_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }; + + let account_filter = AccountFilter { + account: vec![], + owner: vec![], + }; + let trade_event_filter = EventTypeFilter { + include: vec![ + EventType::PumpFunBuy, + EventType::PumpFunSell, + EventType::RaydiumCpmmSwapBaseInput, + EventType::RaydiumCpmmSwapBaseOutput, + ], + }; + + if let Err(e) = client.subscribe_events_immediate( + vec![Protocol::PumpFun, Protocol::RaydiumCpmm], + None, + pumpfun_filter, + account_filter, + Some(trade_event_filter), + None, + callback, + ).await { + println!("Failed to create subscription: {}", e); + return Ok(()); + } + + println!("Subscribed to PumpFun transactions with trade event filters, monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase1_count = event_counter.load(Ordering::Relaxed); + println!("Phase 1: {} events", phase1_count); + + println!("\n=== Phase 2: PumpFun + RaydiumCpmm ==="); + let multi_protocol_filter = TransactionFilter { + account_include: vec![ + PUMPFUN_PROGRAM_ID.to_string(), + RAYDIUM_CPMM_PROGRAM_ID.to_string(), + ], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + multi_protocol_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to PumpFun + RaydiumCpmm transactions, monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase2_count = event_counter.load(Ordering::Relaxed); + println!("Phase 2: {} events", phase2_count - phase1_count); + + println!("\n=== Phase 3: RaydiumCpmm only ==="); + let raydium_cpmm_filter = TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + raydium_cpmm_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to RaydiumCpmm transactions only, monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase3_count = event_counter.load(Ordering::Relaxed); + println!("Phase 3: {} events", phase3_count - phase2_count); + + println!("\n=== Phase 4: Back to PumpFun only ==="); + let pumpfun_only_filter = TransactionFilter { + account_include: vec![PUMPFUN_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + pumpfun_only_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to PumpFun transactions only, monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase4_count = event_counter.load(Ordering::Relaxed); + println!("Phase 4: {} events", phase4_count - phase3_count); + + println!("\n=== Phase 5: All events ==="); + let empty_filter = TransactionFilter { + account_include: vec![], + account_exclude: vec![], + account_required: vec![], + }; + + if let Err(e) = client.update_subscription( + empty_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to all transactions (no filters), monitoring for {}s...", MONITORING_DURATION_SECS); + sleep(Duration::from_secs(MONITORING_DURATION_SECS)).await; + let phase5_count = event_counter.load(Ordering::Relaxed); + println!("Phase 5: {} events", phase5_count - phase4_count); + + println!("\n=== Phase 6: Silence ==="); + + let random_keypair_1 = Keypair::new(); + let random_keypair_2 = Keypair::new(); + let random_pubkey_1 = random_keypair_1.pubkey(); + let random_pubkey_2 = random_keypair_2.pubkey(); + + let silence_filter = TransactionFilter { + account_include: vec![], + account_exclude: vec![], + account_required: vec![ + random_pubkey_1.to_string(), + random_pubkey_2.to_string(), + ], + }; + + if let Err(e) = client.update_subscription( + silence_filter, + AccountFilter { + account: vec![], + owner: vec![], + }, + ).await { + println!("Failed to update subscription: {}", e); + return Ok(()); + } + + println!("Updated to random addresses (expecting silence), monitoring for 3s..."); + let before_silence = event_counter.load(Ordering::Relaxed); + let start_time = Instant::now(); + let last_event_time = Arc::new(Mutex::new(start_time)); + let last_event_time_clone = last_event_time.clone(); + + let mut last_count = before_silence; + for _ in 0..6 { + sleep(Duration::from_millis(500)).await; + let current_count = event_counter.load(Ordering::Relaxed); + if current_count > last_count { + if let Ok(mut time) = last_event_time_clone.lock() { + *time = Instant::now(); + } + last_count = current_count; + } + } + + let final_count = event_counter.load(Ordering::Relaxed); + let events_during_silence = final_count - before_silence; + + if events_during_silence == 0 { + println!("Phase 6: 0 events (immediate filter application)"); + } else if let Ok(last_time) = last_event_time.lock() { + let propagation_time = last_time.duration_since(start_time); + println!("Phase 6: {} events during propagation, filter took {}ms", + events_during_silence, propagation_time.as_millis()); + } + + println!("\n=== Phase 7: Shutdown ==="); + + let shutdown_client = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let shutdown_event_counter = Arc::new(AtomicU64::new(0)); + let shutdown_counter = shutdown_event_counter.clone(); + let shutdown_callback = move |_event: Box| { + shutdown_counter.fetch_add(1, Ordering::Relaxed); + }; + + if let Err(e) = shutdown_client.subscribe_events_immediate( + vec![Protocol::PumpFun, Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + shutdown_callback, + ).await { + println!("Failed to subscribe shutdown client: {}", e); + return Ok(()); + } + + sleep(Duration::from_millis(1000)).await; + let pre_stop_count = shutdown_event_counter.load(Ordering::Relaxed); + println!("Received {} events before stop", pre_stop_count); + + let stop_time = Instant::now(); + shutdown_client.stop().await; + let shutdown_duration = stop_time.elapsed(); + println!("stop() completed in {:.1}ms", shutdown_duration.as_millis()); + + let post_stop_count = shutdown_event_counter.load(Ordering::Relaxed); + let during_stop = post_stop_count - pre_stop_count; + if during_stop > 0 { + println!(" {} events received during stop()", during_stop); + } + + let last_event_time = Arc::new(Mutex::new(stop_time)); + let last_event_time_clone = last_event_time.clone(); + let mut last_count = post_stop_count; + + for _ in 0..20 { + sleep(Duration::from_millis(100)).await; + let current_count = shutdown_event_counter.load(Ordering::Relaxed); + if current_count > last_count { + if let Ok(mut time) = last_event_time_clone.lock() { + *time = Instant::now(); + } + last_count = current_count; + } + } + + let final_count = shutdown_event_counter.load(Ordering::Relaxed); + let after_stop = final_count - post_stop_count; + + if after_stop == 0 { + println!("Phase 7: Clean shutdown - no events after stop()"); + } else if let Ok(last_time) = last_event_time.lock() { + let post_stop_duration = last_time.duration_since(stop_time); + let silence_duration = Instant::now().duration_since(*last_time); + println!("Phase 7: {} events arrived up to {}ms after stop(), then silent for {}ms", + after_stop, post_stop_duration.as_millis(), silence_duration.as_millis()); + } + + println!("\n=== Subscription enforcement ==="); + + let test_callback = |_event: Box| {}; + + match client.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + test_callback, + ).await { + Ok(_) => println!("ERROR: Same client created second subscription"), + Err(e) if e.to_string().contains("Already subscribed") => { + println!("✓ Single subscription enforcement working"); + }, + Err(e) => println!("Unexpected error: {}", e), + } + + let client2 = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let client2_counter = Arc::new(AtomicU64::new(0)); + let counter2 = client2_counter.clone(); + let client2_callback = move |_event: Box| { + counter2.fetch_add(1, Ordering::Relaxed); + }; + + match client2.subscribe_events_immediate( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + client2_callback, + ).await { + Ok(_) => { + sleep(Duration::from_millis(500)).await; + let count = client2_counter.load(Ordering::Relaxed); + println!("✓ Second client: {} events", count); + client2.stop().await; + }, + Err(e) => println!("ERROR: Second client failed: {}", e), + } + + println!("\n=== Advanced subscription enforcement ==="); + + let test_callback_advanced = |_event: Box| {}; + + let client3 = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + // First subscription should succeed + match client3.subscribe_events_advanced( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + test_callback_advanced, + ).await { + Ok(_) => { + // Second subscription attempt on same client should fail + match client3.subscribe_events_advanced( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + |_| {}, + ).await { + Ok(_) => println!("ERROR: Same client created second advanced subscription"), + Err(e) if e.to_string().contains("Already subscribed") => { + println!("✓ Advanced single subscription enforcement working"); + }, + Err(e) => println!("Unexpected error: {}", e), + } + }, + Err(e) => println!("ERROR: First advanced subscription failed: {}", e), + } + + // Test that a second client can subscribe using advanced method + let client4 = Arc::new(YellowstoneGrpc::new( + GRPC_ENDPOINT.to_string(), + API_KEY.map(|s| s.to_string()) + )?); + + let client4_counter = Arc::new(AtomicU64::new(0)); + let counter4 = client4_counter.clone(); + let client4_callback = move |_event: Box| { + counter4.fetch_add(1, Ordering::Relaxed); + }; + + match client4.subscribe_events_advanced( + vec![Protocol::RaydiumCpmm], + None, + TransactionFilter { + account_include: vec![RAYDIUM_CPMM_PROGRAM_ID.to_string()], + account_exclude: vec![], + account_required: vec![], + }, + AccountFilter { + account: vec![], + owner: vec![], + }, + None, + None, + client4_callback, + ).await { + Ok(_) => { + sleep(Duration::from_millis(500)).await; + let count = client4_counter.load(Ordering::Relaxed); + println!("✓ Second client (advanced): {} events", count); + client4.stop().await; + }, + Err(e) => println!("ERROR: Second client (advanced) failed: {}", e), + } + + client3.stop().await; + + client.stop().await; + + Ok(()) +} diff --git a/src/streaming/event_parser/common/filter.rs b/src/streaming/event_parser/common/filter.rs index 8fa3296..2ac0743 100644 --- a/src/streaming/event_parser/common/filter.rs +++ b/src/streaming/event_parser/common/filter.rs @@ -2,7 +2,7 @@ use crate::streaming::event_parser::common::{ types::EventType, ACCOUNT_EVENT_TYPES, BLOCK_EVENT_TYPES, }; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct EventTypeFilter { pub include: Vec, } diff --git a/src/streaming/grpc/subscription.rs b/src/streaming/grpc/subscription.rs index 2efdddd..15579fa 100644 --- a/src/streaming/grpc/subscription.rs +++ b/src/streaming/grpc/subscription.rs @@ -49,6 +49,7 @@ impl SubscriptionManager { ) -> AnyResult<( impl Sink, impl Stream>, + SubscribeRequest, )> { let blocks_meta = if event_type_filter.is_some() && event_type_filter.as_ref().unwrap().include_block_event() @@ -71,8 +72,8 @@ impl SubscriptionManager { ..Default::default() }; let mut client = self.connect().await?; - let (sink, stream) = client.subscribe_with_request(Some(subscribe_request)).await?; - Ok((sink, stream)) + let (sink, stream) = client.subscribe_with_request(Some(subscribe_request.clone())).await?; + Ok((sink, stream, subscribe_request)) } /// 创建账户订阅请求并返回流 diff --git a/src/streaming/yellowstone_grpc.rs b/src/streaming/yellowstone_grpc.rs index f88e88b..d5ffdd3 100644 --- a/src/streaming/yellowstone_grpc.rs +++ b/src/streaming/yellowstone_grpc.rs @@ -1,10 +1,12 @@ -use futures::{channel::mpsc, StreamExt}; +use futures::{channel::mpsc, SinkExt, StreamExt}; use log::error; use solana_sdk::pubkey::Pubkey; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use tokio::sync::Mutex; -use yellowstone_grpc_proto::geyser::CommitmentLevel; +use tokio::sync::{Mutex, RwLock}; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest}; +use anyhow::anyhow; use crate::common::AnyResult; use crate::streaming::common::{ EventBatchProcessor, MetricsManager, PerformanceMetrics, StreamClientConfig, SubscriptionHandle, @@ -14,6 +16,7 @@ use crate::streaming::event_parser::{Protocol, UnifiedEvent}; use crate::streaming::grpc::{EventPretty, EventProcessor, StreamHandler, SubscriptionManager}; /// 交易过滤器 +#[derive(Debug, Clone)] pub struct TransactionFilter { pub account_include: Vec, pub account_exclude: Vec, @@ -21,6 +24,7 @@ pub struct TransactionFilter { } /// 账户过滤器 +#[derive(Debug, Clone)] pub struct AccountFilter { pub account: Vec, pub owner: Vec, @@ -35,6 +39,10 @@ pub struct YellowstoneGrpc { pub metrics_manager: MetricsManager, pub event_processor: EventProcessor, pub subscription_handle: Arc>>, + // Dynamic subscription management fields + pub active_subscription: Arc, + pub control_tx: Arc>>>, + pub current_request: Arc>>, } impl YellowstoneGrpc { @@ -68,6 +76,9 @@ impl YellowstoneGrpc { metrics_manager, event_processor, subscription_handle: Arc::new(Mutex::new(None)), + active_subscription: Arc::new(AtomicBool::new(false)), + control_tx: Arc::new(Mutex::new(None)), + current_request: Arc::new(RwLock::new(None)), }) } @@ -119,6 +130,10 @@ impl YellowstoneGrpc { if let Some(handle) = handle_guard.take() { handle.stop(); } + + *self.control_tx.lock().await = None; + *self.current_request.write().await = None; + self.active_subscription.store(false, Ordering::Release); } /// Simplified immediate event subscription (recommended for simple scenarios) @@ -147,8 +162,14 @@ impl YellowstoneGrpc { where F: Fn(Box) + Send + Sync + 'static, { - // 如果已有活跃订阅，先停止它 - self.stop().await; + if self.active_subscription.compare_exchange( + false, + true, + Ordering::Acquire, + Ordering::Relaxed, + ).is_err() { + return Err(anyhow!("Already subscribed. Use update_subscription() to modify filters")); + } let mut metrics_handle = None; // 启动自动性能监控（如果启用） @@ -169,35 +190,53 @@ impl YellowstoneGrpc { ); // 订阅事件 - let (mut subscribe_tx, mut stream) = self + let (mut subscribe_tx, mut stream, subscribe_request) = self .subscription_manager .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) .await?; + *self.current_request.write().await = Some(subscribe_request); + + let (control_tx, mut control_rx) = mpsc::channel(100); + + *self.control_tx.lock().await = Some(control_tx); + // 创建通道，使用配置中的通道大小 let (mut tx, mut rx) = mpsc::channel::(self.config.backpressure.channel_size); // 启动流处理任务 let backpressure_strategy = self.config.backpressure.strategy; let stream_handle = tokio::spawn(async move { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - backpressure_strategy, - ) - .await - { - error!("Error handling message: {e:?}"); - break; + loop { + tokio::select! { + message = stream.next() => { + match message { + Some(Ok(msg)) => { + if let Err(e) = StreamHandler::handle_stream_message( + msg, + &mut tx, + &mut subscribe_tx, + backpressure_strategy, + ) + .await + { + error!("Error handling message: {e:?}"); + break; + } + } + Some(Err(error)) => { + error!("Stream error: {error:?}"); + break; + } + None => break, } } - Err(error) => { - error!("Stream error: {error:?}"); - break; + + Some(update) = control_rx.next() => { + if let Err(e) = subscribe_tx.send(update).await { + error!("Failed to send subscription update: {}", e); + break; + } } } } @@ -263,8 +302,15 @@ impl YellowstoneGrpc { where F: Fn(Box) + Send + Sync + 'static, { - // 如果已有活跃订阅，先停止它 - self.stop().await; + // Single subscription enforcement + if self.active_subscription.compare_exchange( + false, + true, + Ordering::Acquire, + Ordering::Relaxed, + ).is_err() { + return Err(anyhow!("Already subscribed. Use update_subscription() to modify filters")); + } let mut metrics_handle = None; // 启动自动性能监控（如果启用） @@ -285,11 +331,17 @@ impl YellowstoneGrpc { ); // Subscribe to events - let (mut subscribe_tx, mut stream) = self + let (mut subscribe_tx, mut stream, subscribe_request) = self .subscription_manager .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) .await?; + *self.current_request.write().await = Some(subscribe_request); + + let (control_tx, mut control_rx) = mpsc::channel(100); + + *self.control_tx.lock().await = Some(control_tx); + // Create channel let (mut tx, mut rx) = mpsc::channel::(self.config.backpressure.channel_size); @@ -309,24 +361,36 @@ impl YellowstoneGrpc { // Start task to process the stream let backpressure_strategy = self.config.backpressure.strategy; let stream_handle = tokio::spawn(async move { - while let Some(message) = stream.next().await { - match message { - Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - backpressure_strategy, - ) - .await - { - error!("Error handling message: {e:?}"); - break; + loop { + tokio::select! { + message = stream.next() => { + match message { + Some(Ok(msg)) => { + if let Err(e) = StreamHandler::handle_stream_message( + msg, + &mut tx, + &mut subscribe_tx, + backpressure_strategy, + ) + .await + { + error!("Error handling message: {e:?}"); + break; + } + } + Some(Err(error)) => { + error!("Stream error: {error:?}"); + break; + } + None => break, } } - Err(error) => { - error!("Stream error: {error:?}"); - break; + + Some(update) = control_rx.next() => { + if let Err(e) = subscribe_tx.send(update).await { + error!("Failed to send subscription update: {}", e); + break; + } } } } @@ -362,6 +426,61 @@ impl YellowstoneGrpc { Ok(()) } + + /// Update subscription filters at runtime without reconnection + /// + /// # Parameters + /// * `transaction_filter` - New transaction filter to apply + /// * `account_filter` - New account filter to apply + /// + /// # Returns + /// Returns `AnyResult<()>` on success, error on failure + pub async fn update_subscription( + &self, + transaction_filter: TransactionFilter, + account_filter: AccountFilter, + ) -> AnyResult<()> { + let mut control_sender = { + let control_guard = self.control_tx.lock().await; + + if !self.active_subscription.load(Ordering::Acquire) { + return Err(anyhow!("No active subscription to update")); + } + + control_guard.as_ref() + .ok_or_else(|| anyhow!("No active subscription to update"))? + .clone() + }; + + let mut request = self.current_request.read().await + .as_ref() + .ok_or_else(|| anyhow!("No active subscription"))? + .clone(); + + request.transactions = self.subscription_manager + .get_subscribe_request_filter( + transaction_filter.account_include, + transaction_filter.account_exclude, + transaction_filter.account_required, + None, + ) + .unwrap_or_default(); + + request.accounts = self.subscription_manager + .subscribe_with_account_request( + account_filter.account, + account_filter.owner, + None, + ) + .unwrap_or_default(); + + control_sender.send(request.clone()).await + .map_err(|e| anyhow!("Failed to send update: {}", e))?; + + *self.current_request.write().await = Some(request); + + Ok(()) + } } // 实现 Clone trait 以支持模块间共享 @@ -376,6 +495,9 @@ impl Clone for YellowstoneGrpc { metrics_manager: self.metrics_manager.clone(), event_processor: self.event_processor.clone(), subscription_handle: self.subscription_handle.clone(), // 共享同一个 Arc> + active_subscription: self.active_subscription.clone(), + control_tx: self.control_tx.clone(), + current_request: self.current_request.clone(), } } } diff --git a/src/streaming/yellowstone_sub_system.rs b/src/streaming/yellowstone_sub_system.rs index ebb4aa9..c47d36f 100755 --- a/src/streaming/yellowstone_sub_system.rs +++ b/src/streaming/yellowstone_sub_system.rs @@ -47,7 +47,7 @@ impl YellowstoneGrpc { addrs, None, ); - let (mut subscribe_tx, mut stream) = self + let (mut subscribe_tx, mut stream, _) = self .subscription_manager .subscribe_with_request(transactions, None, None, None) .await?; From 9ea4dab4df16e8a6a198c6899637707b89b6e965 Mon Sep 17 00:00:00 2001 From: ysq Date: Wed, 27 Aug 2025 21:31:31 +0800 Subject: [PATCH 4/7] perf: Major event processing system refactor for improved performance --- src/main.rs | 26 +- src/streaming/common/constants.rs | 2 +- src/streaming/common/event_processor.rs | 144 ++-- src/streaming/common/metrics.rs | 695 ++++++++++++------ src/streaming/event_parser/common/mod.rs | 9 +- src/streaming/event_parser/common/types.rs | 61 +- src/streaming/event_parser/core/traits.rs | 245 +++--- .../protocols/block/block_meta_event.rs | 3 +- .../event_parser/protocols/bonk/parser.rs | 12 +- .../event_parser/protocols/mutil/parser.rs | 12 +- .../event_parser/protocols/pumpfun/parser.rs | 12 +- .../event_parser/protocols/pumpswap/parser.rs | 12 +- .../protocols/raydium_amm_v4/parser.rs | 12 +- .../protocols/raydium_clmm/parser.rs | 12 +- .../protocols/raydium_cpmm/parser.rs | 12 +- src/streaming/grpc/stream_handler.rs | 87 +-- src/streaming/grpc/types.rs | 13 +- src/streaming/mod.rs | 4 +- src/streaming/shred/connection.rs | 4 +- src/streaming/shred_stream.rs | 12 +- src/streaming/yellowstone_grpc.rs | 18 +- src/streaming/yellowstone_sub_system.rs | 32 +- 22 files changed, 829 insertions(+), 610 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1f6fabc..b95723c 100755 --- a/src/main.rs +++ b/src/main.rs @@ -86,13 +86,17 @@ async fn test_grpc() -> Result<(), Box> { println!("Protocols to monitor: {:?}", protocols); + const SYSTEM_PROGRAM_ID: solana_sdk::pubkey::Pubkey = + solana_sdk::pubkey!("11111111111111111111111111111111"); + // Filter accounts let account_include = vec![ - PUMPFUN_PROGRAM_ID.to_string(), // Listen to pumpfun program ID - PUMPSWAP_PROGRAM_ID.to_string(), // Listen to pumpswap program ID - BONK_PROGRAM_ID.to_string(), // Listen to bonk program ID - RAYDIUM_CPMM_PROGRAM_ID.to_string(), // Listen to raydium_cpmm program ID - RAYDIUM_CLMM_PROGRAM_ID.to_string(), // Listen to raydium_clmm program ID + SYSTEM_PROGRAM_ID.to_string(), + PUMPFUN_PROGRAM_ID.to_string(), // Listen to pumpfun program ID + PUMPSWAP_PROGRAM_ID.to_string(), // Listen to pumpswap program ID + BONK_PROGRAM_ID.to_string(), // Listen to bonk program ID + RAYDIUM_CPMM_PROGRAM_ID.to_string(), // Listen to raydium_cpmm program ID + RAYDIUM_CLMM_PROGRAM_ID.to_string(), // Listen to raydium_clmm program ID RAYDIUM_AMM_V4_PROGRAM_ID.to_string(), // Listen to raydium_amm_v4 program ID ]; let account_exclude = vec![]; @@ -151,7 +155,7 @@ async fn test_shreds() -> Result<(), Box> { // Enable performance monitoring, has performance overhead, disabled by default config.enable_metrics = true; let shred_stream = - ShredStreamGrpc::new_with_config("http://127.0.0.1:10800".to_string(), config).await?; + ShredStreamGrpc::new_with_config("http://64.130.37.195:10800".to_string(), config).await?; let callback = create_event_callback(); let protocols = vec![ @@ -188,17 +192,11 @@ async fn test_shreds() -> Result<(), Box> { fn create_event_callback() -> impl Fn(Box) { |event: Box| { - println!( - "🎉 Event received! Type: {:?}, ID: {}, Slot: {}, Transaction Index: {:?}", - event.event_type(), - event.id(), - event.slot(), - event.transaction_index(), - ); + println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id()); match_event!(event, { // -------------------------- block meta ----------------------- BlockMetaEvent => |e: BlockMetaEvent| { - println!("BlockMetaEvent: {e:?}"); + println!("BlockMetaEvent: {:?}", e.metadata.program_handle_time_consuming_us); }, // -------------------------- bonk ----------------------- BonkPoolCreateEvent => |e: BonkPoolCreateEvent| { diff --git a/src/streaming/common/constants.rs b/src/streaming/common/constants.rs index 8cc2be8..0b253f6 100644 --- a/src/streaming/common/constants.rs +++ b/src/streaming/common/constants.rs @@ -11,4 +11,4 @@ pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5; // 性能监控相关常量 pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5; pub const DEFAULT_METRICS_PRINT_INTERVAL_SECONDS: u64 = 10; -pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 500.0; +pub const SLOW_PROCESSING_THRESHOLD_US: f64 = 3000.0; diff --git a/src/streaming/common/event_processor.rs b/src/streaming/common/event_processor.rs index 7bbacd2..a2364fe 100644 --- a/src/streaming/common/event_processor.rs +++ b/src/streaming/common/event_processor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use tokio::task::JoinHandle; use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Signature; use crate::common::AnyResult; use crate::streaming::common::{ @@ -14,7 +14,7 @@ use crate::streaming::event_parser::EventParser; use crate::streaming::event_parser::{ core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, }; -use crate::streaming::grpc::{BackpressureStrategy, BatchConfig, EventPretty}; +use crate::streaming::grpc::{BackpressureConfig, BatchConfig, EventPretty}; use crate::streaming::shred::TransactionWithSlot; use once_cell::sync::OnceCell; @@ -25,21 +25,24 @@ pub struct EventProcessor { pub(crate) parser_cache: OnceCell>, pub(crate) protocols: Vec, pub(crate) event_type_filter: Option, - pub(crate) backpressure_strategy: BackpressureStrategy, + pub(crate) callback: Option) + Send + Sync>>, + pub(crate) backpressure_config: BackpressureConfig, pub(crate) batch_config: BatchConfig, } impl EventProcessor { /// 创建新的事件处理器 pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { + let backpressure_config = config.backpressure.clone(); Self { metrics_manager, config, parser_cache: OnceCell::new(), protocols: vec![], event_type_filter: None, - backpressure_strategy: BackpressureStrategy::Block, + backpressure_config, batch_config: BatchConfig::default(), + callback: None, } } @@ -47,13 +50,15 @@ impl EventProcessor { &mut self, protocols: Vec, event_type_filter: Option, - backpressure_strategy: BackpressureStrategy, + backpressure_config: BackpressureConfig, batch_config: BatchConfig, + callback: Option) + Send + Sync>>, ) { self.protocols = protocols.clone(); self.event_type_filter = event_type_filter.clone(); - self.backpressure_strategy = backpressure_strategy; + self.backpressure_config = backpressure_config; self.batch_config = batch_config; + self.callback = callback; self.parser_cache .get_or_init(|| Arc::new(MutilEventParser::new(protocols, event_type_filter))); } @@ -62,35 +67,24 @@ impl EventProcessor { self.parser_cache.get().unwrap().clone() } - pub fn get_event_handle(&self) -> Option> { - return None; - } - - pub async fn process_grpc_event_transaction_with_metrics( + pub async fn process_grpc_event_transaction_with_metrics( &self, event_pretty: EventPretty, - callback: &F, bot_wallet: Option, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { - self.process_grpc_event_transaction(event_pretty, callback, bot_wallet).await?; + ) -> AnyResult<()> { + self.process_grpc_event_transaction(event_pretty, bot_wallet).await?; Ok(()) } - async fn process_grpc_event_transaction( + async fn process_grpc_event_transaction( &self, event_pretty: EventPretty, - callback: &F, bot_wallet: Option, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { + ) -> AnyResult<()> { match event_pretty { EventPretty::Account(account_pretty) => { self.metrics_manager.add_account_process_count(); + let signature = account_pretty.signature; let account_event = AccountEventParser::parse_account_event( self.protocols.clone(), account_pretty, @@ -98,12 +92,12 @@ impl EventProcessor { ); if let Some(event) = account_event { let processing_time_us = event.program_handle_time_consuming_us() as f64; - callback(event); - // 更新性能指标（如果启用） - self.metrics_manager.update_metrics( + self.invoke_callback(event); + self.update_metrics( MetricsEventType::Account, 1, processing_time_us, + Some(signature), ); } } @@ -113,7 +107,7 @@ impl EventProcessor { let signature = transaction_pretty.signature; // 使用缓存获取解析器 let parser = self.get_parser(); - let mut all_events = parser + let all_events = parser .parse_transaction( transaction_pretty.tx.clone(), signature, @@ -125,37 +119,26 @@ impl EventProcessor { .await .unwrap_or_else(|_e| vec![]); - // 为所有事件设置交易索引 - for event in &mut all_events { - event.set_transaction_index(transaction_pretty.transaction_index); - } - - let max_time_consuming_us = all_events - .iter() - .map(|event| event.program_handle_time_consuming_us()) - .max() - .unwrap_or(0); - - // 保存事件数量用于日志记录 + let mut max_time_consuming_us = 0; let event_count = all_events.len(); - // 批量处理事件 - if !all_events.is_empty() { - for mut event in all_events { - event.set_program_handle_time_consuming_us( - chrono::Utc::now().timestamp_micros() - - event.program_received_time_us(), - ); - callback(event); - } + // 为所有事件设置交易索引 + for mut event in all_events { + event.set_transaction_index(transaction_pretty.transaction_index); + event.set_program_handle_time_consuming_us( + chrono::Utc::now().timestamp_micros() - event.program_received_time_us(), + ); + max_time_consuming_us = + max_time_consuming_us.max(event.program_handle_time_consuming_us()); + self.invoke_callback(event); } // 更新性能指标 - // 更新性能指标（如果启用） - self.metrics_manager.update_metrics( - MetricsEventType::Tx, + self.update_metrics( + MetricsEventType::Transaction, event_count as u64, max_time_consuming_us as f64, + Some(signature), ); } EventPretty::BlockMeta(block_meta_pretty) => { @@ -171,29 +154,34 @@ impl EventProcessor { block_meta_pretty.program_received_time_us, ); let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64; - callback(block_meta_event); - // 更新性能指标（如果启用） - self.metrics_manager.update_metrics( - MetricsEventType::BlockMeta, - 1, - processing_time_us, - ); + self.invoke_callback(block_meta_event); + self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us, None); } } Ok(()) } + pub fn invoke_callback(&self, event: Box) { + if let Some(callback) = self.callback.as_ref() { + callback(event); + } + } + /// 即时处理单个交易 - pub async fn process_shred_transaction_immediate( + pub async fn process_shred_transaction_immediate( &self, transaction_with_slot: TransactionWithSlot, bot_wallet: Option, - callback: &F, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { + ) -> AnyResult<()> { + self.process_shred_transaction(transaction_with_slot, bot_wallet).await + } + + pub async fn process_shred_transaction( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { self.metrics_manager.add_tx_process_count(); let program_received_time_us = chrono::Utc::now().timestamp_micros(); let slot = transaction_with_slot.slot; @@ -215,11 +203,7 @@ impl EventProcessor { .await .unwrap_or_else(|_e| vec![]); - let max_time_consuming_us = all_events - .iter() - .map(|event| event.program_handle_time_consuming_us()) - .max() - .unwrap_or(0); + let mut max_time_consuming_us = 0; // 保存事件数量用于日志记录 let event_count = all_events.len(); @@ -229,18 +213,31 @@ impl EventProcessor { event.set_program_handle_time_consuming_us( chrono::Utc::now().timestamp_micros() - event.program_received_time_us(), ); - callback(event); + max_time_consuming_us = + max_time_consuming_us.max(event.program_handle_time_consuming_us()); + self.invoke_callback(event); } // 实际调用性能指标更新 - self.metrics_manager.update_metrics( - MetricsEventType::Tx, + self.update_metrics( + MetricsEventType::Transaction, event_count as u64, max_time_consuming_us as f64, + Some(signature), ); Ok(()) } + + fn update_metrics( + &self, + ty: MetricsEventType, + count: u64, + time_us: f64, + signature: Option, + ) { + self.metrics_manager.update_metrics(ty, count, time_us, signature); + } } // 实现 Clone trait 以支持模块间共享 @@ -252,8 +249,9 @@ impl Clone for EventProcessor { parser_cache: self.parser_cache.clone(), protocols: self.protocols.clone(), event_type_filter: self.event_type_filter.clone(), - backpressure_strategy: self.backpressure_strategy.clone(), + backpressure_config: self.backpressure_config.clone(), batch_config: self.batch_config.clone(), + callback: self.callback.clone(), } } } diff --git a/src/streaming/common/metrics.rs b/src/streaming/common/metrics.rs index 6dd80e5..a5df787 100644 --- a/src/streaming/common/metrics.rs +++ b/src/streaming/common/metrics.rs @@ -1,344 +1,554 @@ +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; -use crossbeam::utils::Backoff; -use crossbeam::atomic::AtomicCell; -use std::sync::RwLock; -use super::config::StreamClientConfig; +use solana_sdk::signature::Signature; + use super::constants::*; -/// 单个事件类型的指标 +/// 事件类型枚举 +#[derive(Debug, Clone, Copy)] +pub enum EventType { + Transaction = 0, + Account = 1, + BlockMeta = 2, +} + +/// 兼容性别名 +pub type MetricsEventType = EventType; + +impl EventType { + #[inline] + const fn as_index(self) -> usize { + self as usize + } + + const fn name(self) -> &'static str { + match self { + EventType::Transaction => "TX", + EventType::Account => "Account", + EventType::BlockMeta => "Block Meta", + } + } + + // 兼容性常量 + pub const TX: EventType = EventType::Transaction; +} + +/// 高性能原子事件指标 +#[derive(Debug)] +struct AtomicEventMetrics { + process_count: AtomicU64, + events_processed: AtomicU64, + events_in_window: AtomicU64, + window_start_nanos: AtomicU64, + events_per_second_bits: AtomicU64, // f64 的位表示 +} + +impl AtomicEventMetrics { + fn new(now_nanos: u64) -> Self { + Self { + process_count: AtomicU64::new(0), + events_processed: AtomicU64::new(0), + events_in_window: AtomicU64::new(0), + window_start_nanos: AtomicU64::new(now_nanos), + events_per_second_bits: AtomicU64::new(0), + } + } + + /// 原子地增加处理计数 + #[inline] + fn add_process_count(&self) { + self.process_count.fetch_add(1, Ordering::Relaxed); + } + + /// 原子地增加事件处理数量 + #[inline] + fn add_events_processed(&self, count: u64) { + self.events_processed.fetch_add(count, Ordering::Relaxed); + self.events_in_window.fetch_add(count, Ordering::Relaxed); + } + + /// 获取当前计数（非阻塞） + #[inline] + fn get_counts(&self) -> (u64, u64, u64) { + ( + self.process_count.load(Ordering::Relaxed), + self.events_processed.load(Ordering::Relaxed), + self.events_in_window.load(Ordering::Relaxed), + ) + } + + /// 原子地更新每秒事件数 + #[inline] + fn update_events_per_second(&self, eps: f64) { + self.events_per_second_bits.store(eps.to_bits(), Ordering::Relaxed); + } + + /// 获取每秒事件数 + #[inline] + fn get_events_per_second(&self) -> f64 { + f64::from_bits(self.events_per_second_bits.load(Ordering::Relaxed)) + } + + /// 重置窗口计数 + #[inline] + fn reset_window(&self, new_start_nanos: u64) { + self.events_in_window.store(0, Ordering::Relaxed); + self.window_start_nanos.store(new_start_nanos, Ordering::Relaxed); + } + + #[inline] + fn get_window_start(&self) -> u64 { + self.window_start_nanos.load(Ordering::Relaxed) + } +} + +/// 高性能原子处理时间统计 +#[derive(Debug)] +struct AtomicProcessingTimeStats { + min_time_bits: AtomicU64, + max_time_bits: AtomicU64, + total_time_us: AtomicU64, // 存储微秒的整数部分 + total_events: AtomicU64, +} + +impl AtomicProcessingTimeStats { + fn new() -> Self { + Self { + min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()), + max_time_bits: AtomicU64::new(0), + total_time_us: AtomicU64::new(0), + total_events: AtomicU64::new(0), + } + } + + /// 原子地更新处理时间统计 + #[inline] + fn update(&self, time_us: f64, event_count: u64) { + let time_bits = time_us.to_bits(); + + // 更新最小值（使用 compare_exchange_weak 循环） + let mut current_min = self.min_time_bits.load(Ordering::Relaxed); + while time_bits < current_min { + match self.min_time_bits.compare_exchange_weak( + current_min, + time_bits, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(x) => current_min = x, + } + } + + // 更新最大值 + let mut current_max = self.max_time_bits.load(Ordering::Relaxed); + while time_bits > current_max { + match self.max_time_bits.compare_exchange_weak( + current_max, + time_bits, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(x) => current_max = x, + } + } + + // 更新累计值（将微秒转换为整数避免浮点累加问题） + let total_time_us_int = (time_us * event_count as f64) as u64; + self.total_time_us.fetch_add(total_time_us_int, Ordering::Relaxed); + self.total_events.fetch_add(event_count, Ordering::Relaxed); + } + + /// 获取统计值（非阻塞） + #[inline] + fn get_stats(&self) -> ProcessingTimeStats { + let min_bits = self.min_time_bits.load(Ordering::Relaxed); + let max_bits = self.max_time_bits.load(Ordering::Relaxed); + let total_time_us_int = self.total_time_us.load(Ordering::Relaxed); + let total_events = self.total_events.load(Ordering::Relaxed); + + let min_time = f64::from_bits(min_bits); + let max_time = f64::from_bits(max_bits); + let avg_time = + if total_events > 0 { total_time_us_int as f64 / total_events as f64 } else { 0.0 }; + + ProcessingTimeStats { + min_us: if min_time == f64::INFINITY { 0.0 } else { min_time }, + max_us: max_time, + avg_us: avg_time, + } + } +} + +/// 处理时间统计结果 #[derive(Debug, Clone)] -pub struct EventMetrics { +pub struct ProcessingTimeStats { + pub min_us: f64, + pub max_us: f64, + pub avg_us: f64, +} + +/// 事件指标快照 +#[derive(Debug, Clone)] +pub struct EventMetricsSnapshot { pub process_count: u64, pub events_processed: u64, pub events_per_second: f64, - pub events_in_window: u64, - pub window_start_time: std::time::Instant, } -impl EventMetrics { - fn new(now: std::time::Instant) -> Self { - Self { - process_count: 0, - events_processed: 0, - events_per_second: 0.0, - events_in_window: 0, - window_start_time: now, - } - } -} - -/// 通用性能监控指标 +/// 兼容性结构 - 完整的性能指标 #[derive(Debug, Clone)] pub struct PerformanceMetrics { - pub start_time: std::time::Instant, - pub event_metrics: [EventMetrics; 3], // [Tx, Account, BlockMeta] - pub average_processing_time_us: f64, - pub min_processing_time_us: f64, - pub max_processing_time_us: f64, - pub last_update_time: std::time::Instant, -} - -impl Default for PerformanceMetrics { - fn default() -> Self { - Self::new() - } -} - -pub enum MetricsEventType { - Tx, - Account, - BlockMeta, -} - -impl MetricsEventType { - fn as_index(&self) -> usize { - match self { - MetricsEventType::Tx => 0, - MetricsEventType::Account => 1, - MetricsEventType::BlockMeta => 2, - } - } + pub uptime: std::time::Duration, + pub tx_metrics: EventMetricsSnapshot, + pub account_metrics: EventMetricsSnapshot, + pub block_meta_metrics: EventMetricsSnapshot, + pub processing_stats: ProcessingTimeStats, } impl PerformanceMetrics { + /// 创建默认的性能指标（兼容性方法） pub fn new() -> Self { - let now = std::time::Instant::now(); + let default_metrics = + EventMetricsSnapshot { process_count: 0, events_processed: 0, events_per_second: 0.0 }; + let default_stats = ProcessingTimeStats { min_us: 0.0, max_us: 0.0, avg_us: 0.0 }; + Self { - start_time: now, - event_metrics: [EventMetrics::new(now), EventMetrics::new(now), EventMetrics::new(now)], - average_processing_time_us: 0.0, - min_processing_time_us: 0.0, - max_processing_time_us: 0.0, - last_update_time: now, + uptime: std::time::Duration::ZERO, + tx_metrics: default_metrics.clone(), + account_metrics: default_metrics.clone(), + block_meta_metrics: default_metrics, + processing_stats: default_stats, + } + } +} + +/// 高性能指标系统 +#[derive(Debug)] +pub struct HighPerformanceMetrics { + start_nanos: u64, + event_metrics: [AtomicEventMetrics; 3], + processing_stats: AtomicProcessingTimeStats, +} + +impl HighPerformanceMetrics { + fn new() -> Self { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + + Self { + start_nanos: now_nanos, + event_metrics: [ + AtomicEventMetrics::new(now_nanos), + AtomicEventMetrics::new(now_nanos), + AtomicEventMetrics::new(now_nanos), + ], + processing_stats: AtomicProcessingTimeStats::new(), } } - /// 更新时间窗口指标 - fn update_window_metrics( - &mut self, - event_type: &MetricsEventType, - now: std::time::Instant, - window_duration: std::time::Duration, - ) { + /// 获取运行时长（秒） + #[inline] + pub fn get_uptime_seconds(&self) -> f64 { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + (now_nanos - self.start_nanos) as f64 / 1_000_000_000.0 + } + + /// 获取事件指标快照 + #[inline] + pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot { let index = event_type.as_index(); - let event_metric = &mut self.event_metrics[index]; + let (process_count, events_processed, _) = self.event_metrics[index].get_counts(); + let events_per_second = self.calculate_real_time_eps(event_type); - if now.duration_since(event_metric.window_start_time) >= window_duration { - let window_seconds = now.duration_since(event_metric.window_start_time).as_secs_f64(); - // 修复：正确计算每秒事件数，避免除零错误 - event_metric.events_per_second = if window_seconds > 0.001 { - // 避免极小的时间差 - event_metric.events_in_window as f64 / window_seconds - } else { - 0.0 // 时间太短时设为0，而不是事件总数 - }; - - // 重置窗口 - event_metric.events_in_window = 0; - event_metric.window_start_time = now; - } + EventMetricsSnapshot { process_count, events_processed, events_per_second } } - /// 计算实时每秒事件数（用于显示） - fn calculate_real_time_events_per_second( - &self, - event_type: &MetricsEventType, - now: std::time::Instant, - ) -> f64 { + /// 获取处理时间统计 + #[inline] + pub fn get_processing_stats(&self) -> ProcessingTimeStats { + self.processing_stats.get_stats() + } + + /// 计算实时每秒事件数（非阻塞） + fn calculate_real_time_eps(&self, event_type: EventType) -> f64 { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + let index = event_type.as_index(); let event_metric = &self.event_metrics[index]; - let current_window_duration = - now.duration_since(event_metric.window_start_time).as_secs_f64(); + let window_start = event_metric.get_window_start(); + let current_window_duration_secs = + (now_nanos.saturating_sub(window_start)) as f64 / 1_000_000_000.0; + let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed); - // 如果当前窗口有足够的时间和事件，使用当前窗口的数据 - if current_window_duration > 1.0 && event_metric.events_in_window > 0 { - event_metric.events_in_window as f64 / current_window_duration + // 优先级1: 当前窗口实时数据（≥2秒且有事件） + if current_window_duration_secs >= 2.0 && events_in_window > 0 { + return events_in_window as f64 / current_window_duration_secs; } - // 如果当前窗口时间太短或没有事件，使用上一个完整窗口的值 - else if event_metric.events_per_second > 0.0 { - event_metric.events_per_second + + // 优先级2: 上一个窗口的结果 + let stored_eps = event_metric.get_events_per_second(); + if stored_eps > 0.0 { + return stored_eps; } - // 如果都没有，计算总体平均值 - else { - let total_duration = now.duration_since(self.start_time).as_secs_f64(); - if total_duration > 1.0 && event_metric.events_processed > 0 { - event_metric.events_processed as f64 / total_duration - } else { - 0.0 + + // 优先级3: 总体平均值（≥3秒运行时间） + let total_duration_secs = self.get_uptime_seconds(); + let total_events = event_metric.events_processed.load(Ordering::Relaxed); + if total_duration_secs >= 3.0 && total_events > 0 { + return total_events as f64 / total_duration_secs; + } + + 0.0 + } + + /// 更新窗口指标（后台任务调用） + fn update_window_metrics(&self, event_type: EventType, window_duration_nanos: u64) { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + + let index = event_type.as_index(); + let event_metric = &self.event_metrics[index]; + + let window_start = event_metric.get_window_start(); + if now_nanos.saturating_sub(window_start) >= window_duration_nanos { + let events_in_window = event_metric.events_in_window.load(Ordering::Relaxed); + let window_duration_secs = window_duration_nanos as f64 / 1_000_000_000.0; + + if window_duration_secs > 0.001 && events_in_window > 0 { + let eps = events_in_window as f64 / window_duration_secs; + event_metric.update_events_per_second(eps); } + + event_metric.reset_window(now_nanos); } } } -/// 通用性能监控管理器 +/// 高性能指标管理器 pub struct MetricsManager { - metrics: Arc>, - config: Arc, + metrics: Arc, + enable_metrics: bool, stream_name: String, + background_task_running: AtomicBool, } impl MetricsManager { - /// 创建新的性能监控管理器 - pub fn new( - metrics: Arc>, - config: Arc, - stream_name: String, - ) -> Self { - Self { metrics, config, stream_name } + /// 创建新的指标管理器 + pub fn new(enable_metrics: bool, stream_name: String) -> Self { + let manager = Self { + metrics: Arc::new(HighPerformanceMetrics::new()), + enable_metrics, + stream_name, + background_task_running: AtomicBool::new(false), + }; + + // 启动后台任务 + manager.start_background_tasks(); + manager } - /// 获取性能指标 - pub fn get_metrics(&self) -> PerformanceMetrics { - // 使用 Backoff 策略进行读取尝试 - let backoff = Backoff::new(); - loop { - match self.metrics.read() { - Ok(metrics) => return metrics.clone(), - Err(_) => { - // 如果获取读锁失败，使用指数退避策略 - backoff.snooze(); - continue; - } + /// 启动后台任务 + fn start_background_tasks(&self) { + if self + .background_task_running + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + if !self.enable_metrics { + return; } + + let metrics = self.metrics.clone(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_millis(500)); + + loop { + interval.tick().await; + + let window_duration_nanos = DEFAULT_METRICS_WINDOW_SECONDS * 1_000_000_000; + + // 更新所有事件类型的窗口指标 + metrics.update_window_metrics(EventType::Transaction, window_duration_nanos); + metrics.update_window_metrics(EventType::Account, window_duration_nanos); + metrics.update_window_metrics(EventType::BlockMeta, window_duration_nanos); + } + }); } } - /// 打印性能指标 - pub fn print_metrics(&self) { - let metrics = self.get_metrics(); - let event_names = ["TX", "Account", "Block Meta"]; - let event_types = - [MetricsEventType::Tx, MetricsEventType::Account, MetricsEventType::BlockMeta]; - let now = std::time::Instant::now(); + /// 记录处理次数（非阻塞） + #[inline] + pub fn record_process(&self, event_type: EventType) { + if self.enable_metrics { + self.metrics.event_metrics[event_type.as_index()].add_process_count(); + } + } + /// 记录事件处理（非阻塞） + #[inline] + pub fn record_events(&self, event_type: EventType, count: u64, processing_time_us: f64) { + if !self.enable_metrics { + return; + } + + // 原子更新事件计数 + self.metrics.event_metrics[event_type.as_index()].add_events_processed(count); + + // 原子更新处理时间统计 + self.metrics.processing_stats.update(processing_time_us, count); + } + + /// 记录慢处理操作 + #[inline] + pub fn log_slow_processing( + &self, + processing_time_us: f64, + event_count: usize, + signature: Option, + ) { + if processing_time_us > SLOW_PROCESSING_THRESHOLD_US { + log::warn!( + "{} slow processing: {:.2}us for {} events, signature: {:?}", + self.stream_name, + processing_time_us, + event_count, + signature + ); + } + } + + /// 获取运行时长 + pub fn get_uptime(&self) -> std::time::Duration { + std::time::Duration::from_secs_f64(self.metrics.get_uptime_seconds()) + } + + /// 获取事件指标 + pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot { + self.metrics.get_event_metrics(event_type) + } + + /// 获取处理时间统计 + pub fn get_processing_stats(&self) -> ProcessingTimeStats { + self.metrics.get_processing_stats() + } + + /// 打印性能指标（非阻塞） + pub fn print_metrics(&self) { println!("\n📊 {} Performance Metrics", self.stream_name); - println!(" Run Time: {:?}", metrics.start_time.elapsed()); - - // 打印表格头部 + println!(" Run Time: {:?}", self.get_uptime()); + + // 打印事件指标表格 println!("┌─────────────┬──────────────┬──────────────────┬─────────────────┐"); println!("│ Event Type │ Process Count│ Events Processed │ Events/Second │"); println!("├─────────────┼──────────────┼──────────────────┼─────────────────┤"); - // 打印每种事件类型的数据 - for (i, name) in event_names.iter().enumerate() { - let event_metric = &metrics.event_metrics[i]; - // 使用实时计算的每秒事件数，而不是窗口更新的值 - let real_time_eps = metrics.calculate_real_time_events_per_second(&event_types[i], now); - + for event_type in [EventType::Transaction, EventType::Account, EventType::BlockMeta] { + let metrics = self.get_event_metrics(event_type); println!( "│ {:11} │ {:12} │ {:16} │ {:13.2} │", - name, - event_metric.process_count, - event_metric.events_processed, - real_time_eps + event_type.name(), + metrics.process_count, + metrics.events_processed, + metrics.events_per_second ); } println!("└─────────────┴──────────────┴──────────────────┴─────────────────┘"); // 打印处理时间统计表格 + let stats = self.get_processing_stats(); println!("\n⏱️ Processing Time Statistics"); println!("┌─────────────────────┬─────────────┐"); println!("│ Metric │ Value (us) │"); println!("├─────────────────────┼─────────────┤"); - println!("│ Average │ {:9.2} │", metrics.average_processing_time_us); - println!("│ Minimum │ {:9.2} │", metrics.min_processing_time_us); - println!("│ Maximum │ {:9.2} │", metrics.max_processing_time_us); + println!("│ Average │ {:9.2} │", stats.avg_us); + println!("│ Minimum │ {:9.2} │", stats.min_us); + println!("│ Maximum │ {:9.2} │", stats.max_us); println!("└─────────────────────┴─────────────┘"); println!(); } /// 启动自动性能监控任务 pub async fn start_auto_monitoring(&self) -> Option> { - // 检查是否启用性能监控 - if !self.config.enable_metrics { - return None; // 如果未启用性能监控，不启动监控任务 + if !self.enable_metrics { + return None; } - let metrics_manager = self.clone(); + let manager = self.clone(); let handle = tokio::spawn(async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_secs( + let mut interval = tokio::time::interval(std::time::Duration::from_secs( DEFAULT_METRICS_PRINT_INTERVAL_SECONDS, )); loop { interval.tick().await; - metrics_manager.print_metrics(); + manager.print_metrics(); } }); Some(handle) } - /// 更新处理次数 - pub fn add_process_count(&self, event_type: MetricsEventType) { - if !self.config.enable_metrics { - return; - } - - // 使用 Backoff 策略进行写入尝试 - let backoff = Backoff::new(); - loop { - match self.metrics.write() { - Ok(mut metrics) => { - metrics.event_metrics[event_type.as_index()].process_count += 1; - break; - }, - Err(_) => { - // 如果获取写锁失败，使用指数退避策略 - backoff.snooze(); - continue; - } - } + // === 兼容性方法 === + + /// 兼容性构造函数 + pub fn new_with_metrics( + _metrics: Arc>, + enable_metrics: bool, + stream_name: String, + ) -> Self { + Self::new(enable_metrics, stream_name) + } + + /// 获取完整的性能指标（兼容性方法） + pub fn get_metrics(&self) -> PerformanceMetrics { + PerformanceMetrics { + uptime: self.get_uptime(), + tx_metrics: self.get_event_metrics(EventType::Transaction), + account_metrics: self.get_event_metrics(EventType::Account), + block_meta_metrics: self.get_event_metrics(EventType::BlockMeta), + processing_stats: self.get_processing_stats(), } } - // 保持向后兼容的方法 + /// 兼容性方法 - 添加交易处理计数 + #[inline] pub fn add_tx_process_count(&self) { - self.add_process_count(MetricsEventType::Tx); + self.record_process(EventType::Transaction); } + /// 兼容性方法 - 添加账户处理计数 + #[inline] pub fn add_account_process_count(&self) { - self.add_process_count(MetricsEventType::Account); + self.record_process(EventType::Account); } + /// 兼容性方法 - 添加区块元数据处理计数 + #[inline] pub fn add_block_meta_process_count(&self) { - self.add_process_count(MetricsEventType::BlockMeta); + self.record_process(EventType::BlockMeta); } - /// 更新性能指标 + /// 兼容性方法 - 更新指标 + #[inline] pub fn update_metrics( &self, event_type: MetricsEventType, events_processed: u64, processing_time_us: f64, + signature: Option, ) { - // 检查是否启用性能监控 - if !self.config.enable_metrics { - return; - } - - // 使用 Backoff 策略进行写入尝试 - let backoff = Backoff::new(); - loop { - match self.metrics.write() { - Ok(mut metrics) => { - let now = std::time::Instant::now(); - let index = event_type.as_index(); - - // 更新事件计数 - metrics.event_metrics[index].events_processed += events_processed; - metrics.event_metrics[index].events_in_window += events_processed; - - metrics.last_update_time = now; - - // 更新处理时间统计 - if processing_time_us < metrics.min_processing_time_us - || metrics.min_processing_time_us == 0.0 - { - metrics.min_processing_time_us = processing_time_us; - } - if processing_time_us > metrics.max_processing_time_us { - metrics.max_processing_time_us = processing_time_us; - } - - // 计算平均处理时间 - 使用增量更新避免重复计算 - let total_events = metrics.event_metrics[index].events_processed; - if total_events > 0 { - let total_events_f64 = total_events as f64; - let old_total = (total_events_f64 - events_processed as f64).max(0.0); - - metrics.average_processing_time_us = if old_total > 0.0 { - (metrics.average_processing_time_us * old_total - + processing_time_us * events_processed as f64) - / total_events_f64 - } else { - processing_time_us - }; - } - - // 更新时间窗口指标 - let window_duration = std::time::Duration::from_secs(DEFAULT_METRICS_WINDOW_SECONDS); - metrics.update_window_metrics(&event_type, now, window_duration); - break; - }, - Err(_) => { - // 如果获取写锁失败，使用指数退避策略 - backoff.snooze(); - continue; - } - } - } - } - - /// 记录慢处理操作 - pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) { - if processing_time_us > SLOW_PROCESSING_THRESHOLD_US { - log::warn!( - "{} slow processing: {processing_time_us}us for {event_count} events", - self.stream_name - ); - } + self.record_events(event_type, events_processed, processing_time_us); + self.log_slow_processing(processing_time_us, events_processed as usize, signature); } } @@ -346,8 +556,9 @@ impl Clone for MetricsManager { fn clone(&self) -> Self { Self { metrics: self.metrics.clone(), - config: self.config.clone(), + enable_metrics: self.enable_metrics, stream_name: self.stream_name.clone(), + background_task_running: AtomicBool::new(false), // 新实例不自动启动后台任务 } } } diff --git a/src/streaming/event_parser/common/mod.rs b/src/streaming/event_parser/common/mod.rs index 3562a92..85c2d90 100755 --- a/src/streaming/event_parser/common/mod.rs +++ b/src/streaming/event_parser/common/mod.rs @@ -54,7 +54,7 @@ macro_rules! impl_unified_event { Box::new(self.clone()) } - fn merge(&mut self, other: Box) { + fn merge(&mut self, other: &dyn $crate::streaming::event_parser::core::traits::UnifiedEvent) { if let Some(_e) = other.as_any().downcast_ref::<$struct_name>() { $( self.$field = _e.$field.clone(); @@ -66,10 +66,13 @@ macro_rules! impl_unified_event { self.metadata.set_swap_data(swap_data); } - fn index(&self) -> String { - self.metadata.index.clone() + fn instruction_outer_index(&self) -> i64 { + self.metadata.instruction_outer_index } + fn instruction_inner_index(&self) -> Option { + self.metadata.instruction_inner_index + } fn transaction_index(&self) -> Option { self.metadata.transaction_index } diff --git a/src/streaming/event_parser/common/types.rs b/src/streaming/event_parser/common/types.rs index e1ab15e..2bc74ba 100755 --- a/src/streaming/event_parser/common/types.rs +++ b/src/streaming/event_parser/common/types.rs @@ -55,36 +55,9 @@ impl EventMetadataPool { } } -/// Transfer data object pool -pub struct TransferDataPool { - pool: Arc>, -} - -impl Default for TransferDataPool { - fn default() -> Self { - Self::new() - } -} - -impl TransferDataPool { - pub fn new() -> Self { - Self { pool: Arc::new(ArrayQueue::new(TRANSFER_DATA_POOL_SIZE)) } - } - - pub fn acquire(&self) -> Option { - self.pool.pop() - } - - pub fn release(&self, transfer_data: TransferData) { - // 如果队列已满，push 会失败，但不会阻塞 - let _ = self.pool.push(transfer_data); - } -} - // Global object pool instances lazy_static::lazy_static! { pub static ref EVENT_METADATA_POOL: EventMetadataPool = EventMetadataPool::new(); - pub static ref TRANSFER_DATA_POOL: TransferDataPool = TransferDataPool::new(); } #[derive( @@ -304,20 +277,6 @@ impl ProtocolInfo { } } -/// Transfer data -#[derive( - Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, -)] -pub struct TransferData { - pub token_program: Pubkey, - pub source: Pubkey, - pub destination: Pubkey, - pub authority: Option, - pub amount: u64, - pub decimals: Option, - pub mint: Option, -} - #[derive( Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, )] @@ -337,7 +296,7 @@ pub struct EventMetadata { pub id: String, pub signature: String, pub slot: u64, - pub transaction_index: Option, // 新增：交易在slot中的索引 + pub transaction_index: Option, // 新增：交易在slot中的索引 pub block_time: i64, pub block_time_ms: i64, pub program_received_time_us: i64, @@ -345,10 +304,9 @@ pub struct EventMetadata { pub protocol: ProtocolType, pub event_type: EventType, pub program_id: Pubkey, - #[deprecated(note = "Please use swap_data instead")] - pub transfer_datas: Vec, pub swap_data: Option, - pub index: String, // 保留原有的指令索引 + pub instruction_outer_index: i64, + pub instruction_inner_index: Option, } impl EventMetadata { @@ -362,14 +320,15 @@ impl EventMetadata { protocol: ProtocolType, event_type: EventType, program_id: Pubkey, - index: String, + instruction_outer_index: i64, + instruction_inner_index: Option, program_received_time_us: i64, ) -> Self { Self { id, signature, slot, - transaction_index: None, // 默认为None，后续设置 + transaction_index: None, // 默认为None，后续设置 block_time, block_time_ms, program_received_time_us, @@ -377,9 +336,9 @@ impl EventMetadata { protocol, event_type, program_id, - transfer_datas: vec![], swap_data: None, - index, + instruction_outer_index, + instruction_inner_index, } } @@ -413,7 +372,7 @@ lazy_static::lazy_static! { /// Parse token transfer data from next instructions pub fn parse_swap_data_from_next_instructions( - event: Box, + event: &dyn UnifiedEvent, inner_instruction: &solana_transaction_status::InnerInstructions, current_index: i8, accounts: &[Pubkey], @@ -435,7 +394,7 @@ pub fn parse_swap_data_from_next_instructions( let mut from_vault: Option = None; let mut to_vault: Option = None; - match_event!(event, { + match_event!(&*event, { BonkTradeEvent => |e: BonkTradeEvent| { user = Some(e.payer); from_mint = Some(e.base_token_mint); diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index 67d6af8..f9c7f4c 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -7,6 +7,7 @@ use solana_sdk::{ use solana_transaction_status::{InnerInstructions, TransactionWithStatusMeta}; use std::collections::HashMap; use std::fmt::Debug; +use std::sync::Arc; use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData}; use crate::streaming::event_parser::protocols::pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}; @@ -17,7 +18,6 @@ use crate::streaming::event_parser::{ pumpfun::{PumpFunCreateTokenEvent, PumpFunTradeEvent}, }, }; -use crate::streaming::shred::MetricsEventType; /// Unified Event Interface - All protocol events must implement this trait pub trait UnifiedEvent: Debug + Send + Sync { @@ -55,7 +55,7 @@ pub trait UnifiedEvent: Debug + Send + Sync { fn clone_boxed(&self) -> Box; /// Merge events (optional implementation) - fn merge(&mut self, _other: Box) { + fn merge(&mut self, _other: &dyn UnifiedEvent) { // Default implementation: no merging operation } @@ -63,7 +63,8 @@ pub trait UnifiedEvent: Debug + Send + Sync { fn set_swap_data(&mut self, swap_data: SwapData); /// Get index - fn index(&self) -> String; + fn instruction_outer_index(&self) -> i64; + fn instruction_inner_index(&self) -> Option; /// Get transaction index in slot fn transaction_index(&self) -> Option; @@ -88,7 +89,8 @@ pub trait EventParser: Send + Sync { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec>; /// 从指令中解析事件数据 @@ -101,7 +103,8 @@ pub trait EventParser: Send + Sync { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec>; /// 从VersionedTransaction中解析指令事件的通用方法 @@ -144,7 +147,8 @@ pub trait EventParser: Send + Sync { slot, block_time, program_received_time_us, - format!("{index}"), + index as i64, + None, ) .await { @@ -156,7 +160,7 @@ pub trait EventParser: Send + Sync { { events.iter_mut().for_each(|event| { let swap_data = parse_swap_data_from_next_instructions( - event.clone_boxed(), + event.as_ref(), inn, -1_i8, &accounts, @@ -217,62 +221,100 @@ pub trait EventParser: Send + Sync { let mut inner_instructions: Vec = vec![]; if let Some(meta) = meta { inner_instructions = meta.inner_instructions.unwrap_or_default(); - for loopup in meta.loaded_addresses.writable { - address_table_lookups.push(loopup); - } - for loopup in meta.loaded_addresses.readonly { - address_table_lookups.push(loopup); - } + address_table_lookups.reserve( + meta.loaded_addresses.writable.len() + meta.loaded_addresses.readonly.len(), + ); + address_table_lookups.extend( + meta.loaded_addresses.writable.into_iter().chain(meta.loaded_addresses.readonly), + ); } - let mut accounts: Vec = vec![]; + + let mut accounts = Vec::with_capacity( + versioned_tx.message.static_account_keys().len() + address_table_lookups.len(), + ); + accounts.extend_from_slice(versioned_tx.message.static_account_keys()); + accounts.extend(address_table_lookups); + + // 使用 Arc 包装共享数据，避免不必要的克隆 + let accounts_arc = Arc::new(accounts); + let inner_instructions_arc = Arc::new(inner_instructions); // 预分配容量，避免动态扩容 let mut instruction_events: Vec> = Vec::with_capacity(16); + let mut inner_instruction_events: Vec> = Vec::with_capacity(8); - // 解析指令事件 - accounts = versioned_tx.message.static_account_keys().to_vec(); - accounts.extend(address_table_lookups.clone()); - - instruction_events = self - .parse_instruction_events_from_versioned_transaction( + let accounts_for_task1 = Arc::clone(&accounts_arc); + let inner_instructions_for_task1 = Arc::clone(&inner_instructions_arc); + let task1 = async move { + self.parse_instruction_events_from_versioned_transaction( &versioned_tx, signature, slot, block_time, program_received_time_us, - &accounts, - &inner_instructions, + &accounts_for_task1, + &inner_instructions_for_task1, ) .await - .unwrap_or_else(|_e| vec![]); + .unwrap_or_else(|_e| vec![]) + }; // 解析内联指令事件 - // 预分配容量，避免动态扩容 - let mut inner_instruction_events: Vec> = Vec::with_capacity(8); - // 检查交易是否成功 - for inner_instruction in inner_instructions { + let inner_instructions_for_task2 = Arc::clone(&inner_instructions_arc); + let accounts_for_task2_1 = Arc::clone(&accounts_arc); + let accounts_for_task2_2 = Arc::clone(&accounts_arc); + + let mut task2_params = Vec::with_capacity(inner_instructions_for_task2.len() * 5); + for inner_instruction in inner_instructions_for_task2.iter() { for (index, instruction) in inner_instruction.instructions.iter().enumerate() { - // 解析嵌套指令 - let compiled_instruction = instruction.instruction.clone(); + task2_params.push(( + &instruction.instruction, + signature, + slot, + block_time, + program_received_time_us, + inner_instruction.index as i64, + Some(index as i64), + inner_instruction, + )); + } + } + // 转换为 Arc<[T]> 更轻量 + let task2_params: Arc<[_]> = task2_params.into(); + let task2_params_clone = Arc::clone(&task2_params); + let task2_1 = async move { + let mut instruction_events: Vec> = Vec::with_capacity(16); + for ( + instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + inner_instruction, + ) in task2_params_clone.iter() + { if let Ok(mut events) = self .parse_instruction( - &compiled_instruction, - &accounts, - signature, - slot, - block_time, - program_received_time_us, - format!("{}.{}", inner_instruction.index, index), + instruction, + &accounts_for_task2_1, + *signature, + *slot, + *block_time, + *program_received_time_us, + *outer_index, + *inner_index, ) .await { if !events.is_empty() { events.iter_mut().for_each(|event| { let swap_data = parse_swap_data_from_next_instructions( - event.clone_boxed(), + event.as_ref(), &inner_instruction, - index as i8, - &accounts, + inner_index.unwrap_or_default() as i8, + &accounts_for_task2_1, ); if let Some(swap_data) = swap_data { event.set_swap_data(swap_data); @@ -281,24 +323,41 @@ pub trait EventParser: Send + Sync { instruction_events.extend(events); } } + } + instruction_events + }; + let task2_2 = async move { + let mut inner_instruction_events: Vec> = Vec::with_capacity(8); + for ( + instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + inner_instruction, + ) in task2_params.iter() + { if let Ok(mut events) = self .parse_inner_instruction( - &compiled_instruction, - signature, - slot, - block_time, - program_received_time_us, - format!("{}.{}", inner_instruction.index, index), + instruction, + *signature, + *slot, + *block_time, + *program_received_time_us, + *outer_index, + *inner_index, ) .await { if !events.is_empty() { events.iter_mut().for_each(|event| { let swap_data = parse_swap_data_from_next_instructions( - event.clone_boxed(), + event.as_ref(), &inner_instruction, - index as i8, - &accounts, + inner_index.unwrap_or_default() as i8, + &accounts_for_task2_2, ); if let Some(swap_data) = swap_data { event.set_swap_data(swap_data); @@ -308,39 +367,41 @@ pub trait EventParser: Send + Sync { } } } - } + inner_instruction_events + }; + + let (r1, r2_1, r2_2) = tokio::join!(task1, task2_1, task2_2); + instruction_events.extend(r1); + instruction_events.extend(r2_1); + inner_instruction_events.extend(r2_2); if !instruction_events.is_empty() && !inner_instruction_events.is_empty() { for instruction_event in &mut instruction_events { for inner_instruction_event in &inner_instruction_events { if instruction_event.id() == inner_instruction_event.id() { - let i_index = instruction_event.index(); - let in_index = inner_instruction_event.index(); - if !i_index.contains(".") && in_index.contains(".") { - let in_index_parts: Vec<&str> = in_index.split(".").collect(); - if !in_index_parts.is_empty() && in_index_parts[0] == i_index { - instruction_event.merge(inner_instruction_event.clone_boxed()); + if instruction_event.instruction_inner_index().is_none() + && inner_instruction_event.instruction_inner_index().is_some() + { + if inner_instruction_event.instruction_outer_index() + == instruction_event.instruction_outer_index() + { + instruction_event.merge(inner_instruction_event.as_ref()); break; } - } else if i_index.contains(".") && in_index.contains(".") { - // 嵌套指令 - let i_index_parts: Vec<&str> = i_index.split(".").collect(); - let in_index_parts: Vec<&str> = in_index.split(".").collect(); - - if !i_index_parts.is_empty() - && !in_index_parts.is_empty() - && i_index_parts[0] == in_index_parts[0] + } else if instruction_event.instruction_inner_index().is_some() + && inner_instruction_event.instruction_inner_index().is_some() + { + if instruction_event.instruction_outer_index() + == inner_instruction_event.instruction_outer_index() { - let i_index_child_index = i_index_parts - .get(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - let in_index_child_index = in_index_parts - .get(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - if in_index_child_index > i_index_child_index { - instruction_event.merge(inner_instruction_event.clone_boxed()); + if inner_instruction_event + .instruction_inner_index() + .unwrap_or_default() + > instruction_event + .instruction_inner_index() + .unwrap_or_default() + { + instruction_event.merge(inner_instruction_event.as_ref()); break; } } @@ -433,7 +494,8 @@ pub trait EventParser: Send + Sync { slot: Option, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Result>> { let slot = slot.unwrap_or(0); let events = self.parse_events_from_inner_instruction( @@ -442,7 +504,8 @@ pub trait EventParser: Send + Sync { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ); Ok(events) } @@ -456,7 +519,8 @@ pub trait EventParser: Send + Sync { slot: Option, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Result>> { let slot = slot.unwrap_or(0); let events = self.parse_events_from_instruction( @@ -466,7 +530,8 @@ pub trait EventParser: Send + Sync { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ); Ok(events) } @@ -543,7 +608,8 @@ impl GenericEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Option> { if let Some(parser) = config.inner_instruction_parser { let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); @@ -557,7 +623,8 @@ impl GenericEventParser { config.protocol_type.clone(), config.event_type.clone(), config.program_id, - index, + outer_index, + inner_index, program_received_time_us, ); parser(data, metadata) @@ -577,7 +644,8 @@ impl GenericEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Option> { if let Some(parser) = config.instruction_parser { let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); @@ -591,7 +659,8 @@ impl GenericEventParser { config.protocol_type.clone(), config.event_type.clone(), config.program_id, - index, + outer_index, + inner_index, program_received_time_us, ); parser(data, account_pubkeys, metadata) @@ -604,9 +673,11 @@ impl GenericEventParser { #[async_trait::async_trait] impl EventParser for GenericEventParser { fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { + // 返回引用而非克隆，减少内存分配 self.inner_instruction_configs.clone() } fn instruction_configs(&self) -> HashMap, Vec> { + // 返回引用而非克隆，减少内存分配 self.instruction_configs.clone() } /// 从内联指令中解析事件数据 @@ -618,7 +689,8 @@ impl EventParser for GenericEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { let inner_instruction_data_decoded = inner_instruction.data.clone(); if inner_instruction_data_decoded.len() < 16 { @@ -638,7 +710,8 @@ impl EventParser for GenericEventParser { slot, block_time, program_received_time_us, - index.clone(), + outer_index, + inner_index, ) { events.push(event); } @@ -658,7 +731,8 @@ impl EventParser for GenericEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { let program_id = accounts[instruction.program_id_index as usize]; if !self.should_handle(&program_id) { @@ -691,7 +765,8 @@ impl EventParser for GenericEventParser { slot, block_time, program_received_time_us, - index.clone(), + outer_index, + inner_index, ) { events.push(event); } diff --git a/src/streaming/event_parser/protocols/block/block_meta_event.rs b/src/streaming/event_parser/protocols/block/block_meta_event.rs index 2987246..2f99ba1 100644 --- a/src/streaming/event_parser/protocols/block/block_meta_event.rs +++ b/src/streaming/event_parser/protocols/block/block_meta_event.rs @@ -28,7 +28,8 @@ impl BlockMetaEvent { crate::streaming::event_parser::common::types::ProtocolType::Common, EventType::BlockMeta, solana_sdk::pubkey::Pubkey::default(), - "".to_string(), + 0, + None, program_received_time_us, ); Self { metadata, slot, block_hash } diff --git a/src/streaming/event_parser/protocols/bonk/parser.rs b/src/streaming/event_parser/protocols/bonk/parser.rs index a03a38e..bc28e85 100755 --- a/src/streaming/event_parser/protocols/bonk/parser.rs +++ b/src/streaming/event_parser/protocols/bonk/parser.rs @@ -618,7 +618,8 @@ impl EventParser for BonkEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -626,7 +627,8 @@ impl EventParser for BonkEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -638,7 +640,8 @@ impl EventParser for BonkEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -647,7 +650,8 @@ impl EventParser for BonkEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/event_parser/protocols/mutil/parser.rs b/src/streaming/event_parser/protocols/mutil/parser.rs index 9748124..b04da73 100755 --- a/src/streaming/event_parser/protocols/mutil/parser.rs +++ b/src/streaming/event_parser/protocols/mutil/parser.rs @@ -79,7 +79,8 @@ impl EventParser for MutilEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -87,7 +88,8 @@ impl EventParser for MutilEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -99,7 +101,8 @@ impl EventParser for MutilEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -108,7 +111,8 @@ impl EventParser for MutilEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/event_parser/protocols/pumpfun/parser.rs b/src/streaming/event_parser/protocols/pumpfun/parser.rs index e907c11..2e97979 100755 --- a/src/streaming/event_parser/protocols/pumpfun/parser.rs +++ b/src/streaming/event_parser/protocols/pumpfun/parser.rs @@ -299,7 +299,8 @@ impl EventParser for PumpFunEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -307,7 +308,8 @@ impl EventParser for PumpFunEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -319,7 +321,8 @@ impl EventParser for PumpFunEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -328,7 +331,8 @@ impl EventParser for PumpFunEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/event_parser/protocols/pumpswap/parser.rs b/src/streaming/event_parser/protocols/pumpswap/parser.rs index 6bd9b57..a688c41 100755 --- a/src/streaming/event_parser/protocols/pumpswap/parser.rs +++ b/src/streaming/event_parser/protocols/pumpswap/parser.rs @@ -389,7 +389,8 @@ impl EventParser for PumpSwapEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -397,7 +398,8 @@ impl EventParser for PumpSwapEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -409,7 +411,8 @@ impl EventParser for PumpSwapEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -418,7 +421,8 @@ impl EventParser for PumpSwapEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs index 593e1af..288e523 100755 --- a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs @@ -391,7 +391,8 @@ impl EventParser for RaydiumAmmV4EventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -399,7 +400,8 @@ impl EventParser for RaydiumAmmV4EventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -411,7 +413,8 @@ impl EventParser for RaydiumAmmV4EventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -420,7 +423,8 @@ impl EventParser for RaydiumAmmV4EventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs index aef8ba9..16fab61 100755 --- a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs @@ -432,7 +432,8 @@ impl EventParser for RaydiumClmmEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -440,7 +441,8 @@ impl EventParser for RaydiumClmmEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -452,7 +454,8 @@ impl EventParser for RaydiumClmmEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -461,7 +464,8 @@ impl EventParser for RaydiumClmmEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs index 6dfdd6b..288bf60 100755 --- a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs @@ -282,7 +282,8 @@ impl EventParser for RaydiumCpmmEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -290,7 +291,8 @@ impl EventParser for RaydiumCpmmEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } @@ -302,7 +304,8 @@ impl EventParser for RaydiumCpmmEventParser { slot: u64, block_time: Option, program_received_time_us: i64, - index: String, + outer_index: i64, + inner_index: Option, ) -> Vec> { self.inner.parse_events_from_instruction( instruction, @@ -311,7 +314,8 @@ impl EventParser for RaydiumCpmmEventParser { slot, block_time, program_received_time_us, - index, + outer_index, + inner_index, ) } diff --git a/src/streaming/grpc/stream_handler.rs b/src/streaming/grpc/stream_handler.rs index ff6fb00..43dbb77 100644 --- a/src/streaming/grpc/stream_handler.rs +++ b/src/streaming/grpc/stream_handler.rs @@ -8,7 +8,6 @@ use yellowstone_grpc_proto::geyser::{ use super::types::{BlockMetaPretty, EventPretty, TransactionPretty}; use crate::common::AnyResult; use crate::streaming::common::EventProcessor; -use crate::streaming::event_parser::UnifiedEvent; use crate::streaming::grpc::AccountPretty; /// 流消息处理器 @@ -16,16 +15,12 @@ pub struct StreamHandler; impl StreamHandler { /// 处理单个流消息 - pub async fn handle_stream_message( + pub async fn handle_stream_message( msg: SubscribeUpdate, subscribe_tx: &mut (impl Sink + Unpin), event_processor: EventProcessor, - callback: &F, bot_wallet: Option, - ) -> AnyResult<()> - where - F: Fn(Box) + Send + Sync, - { + ) -> AnyResult<()> { let created_at = msg.created_at; match msg.update_oneof { Some(UpdateOneof::Account(account)) => { @@ -34,7 +29,6 @@ impl StreamHandler { event_processor .process_grpc_event_transaction_with_metrics( EventPretty::Account(account_pretty), - callback, bot_wallet, ) .await?; @@ -45,7 +39,6 @@ impl StreamHandler { event_processor .process_grpc_event_transaction_with_metrics( EventPretty::BlockMeta(block_meta_pretty), - callback, bot_wallet, ) .await?; @@ -60,7 +53,6 @@ impl StreamHandler { event_processor .process_grpc_event_transaction_with_metrics( EventPretty::Transaction(transaction_pretty), - callback, bot_wallet, ) .await?; @@ -84,58 +76,25 @@ impl StreamHandler { Ok(()) } - // /// 处理背压策略 - // async fn handle_backpressure( - // tx: &mut mpsc::Sender, - // event_pretty: EventPretty, - // backpressure_strategy: BackpressureStrategy, - // ) -> AnyResult<()> { - // match backpressure_strategy { - // BackpressureStrategy::Block => { - // // 阻塞等待，直到有空间 - // if let Err(e) = tx.send(event_pretty).await { - // log::error!("Failed to send transaction to channel: {:?}", e); - // return Err(anyhow::anyhow!("Channel send failed: {:?}", e)); - // } - // } - // BackpressureStrategy::Drop => { - // // 尝试发送，如果失败则丢弃 - // if let Err(e) = tx.try_send(event_pretty) { - // if e.is_full() { - // log::warn!("Channel is full, dropping transaction"); - // } else { - // log::error!("Channel is closed: {:?}", e); - // return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); - // } - // } - // } - // BackpressureStrategy::Retry { max_attempts, wait_ms } => { - // // 重试有限次数 - // let mut retry_count = 0; - // loop { - // match tx.try_send(event_pretty.clone()) { - // Ok(_) => break, - // Err(e) => { - // if e.is_full() { - // retry_count += 1; - // if retry_count >= max_attempts { - // log::warn!( - // "Channel is full after {} attempts, dropping transaction", - // retry_count - // ); - // break; - // } - // tokio::time::sleep(tokio::time::Duration::from_millis(wait_ms)) - // .await; - // } else { - // log::error!("Channel is closed: {:?}", e); - // return Err(anyhow::anyhow!("Channel is closed: {:?}", e)); - // } - // } - // } - // } - // } - // } - // Ok(()) - // } + pub async fn handle_stream_system_message( + msg: SubscribeUpdate, + subscribe_tx: &mut (impl Sink + Unpin), + ) -> AnyResult> { + let created_at = msg.created_at; + let event_pretty = match msg.update_oneof { + Some(UpdateOneof::Transaction(sut)) => Some(TransactionPretty::from((sut, created_at))), + Some(UpdateOneof::Ping(_)) => { + subscribe_tx + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: 1 }), + ..Default::default() + }) + .await?; + None + } + Some(UpdateOneof::Pong(_)) => None, + _ => None, + }; + Ok(event_pretty.map(|e| EventPretty::Transaction(e))) + } } diff --git a/src/streaming/grpc/types.rs b/src/streaming/grpc/types.rs index 9238f9f..7cfd946 100644 --- a/src/streaming/grpc/types.rs +++ b/src/streaming/grpc/types.rs @@ -69,7 +69,7 @@ impl fmt::Debug for BlockMetaPretty { #[derive(Clone)] pub struct TransactionPretty { pub slot: u64, - pub transaction_index: Option, // 新增：交易在slot中的索引 + pub transaction_index: Option, // 新增：交易在slot中的索引 pub block_hash: String, pub block_time: Option, pub signature: Signature, @@ -95,10 +95,11 @@ impl From for AccountPretty { let account_info = account.account.unwrap(); Self { slot: account.slot, - signature: Signature::try_from( - account_info.txn_signature.unwrap_or_default().as_slice(), - ) - .expect("valid signature"), + signature: if let Some(txn_signature) = account_info.txn_signature { + Signature::try_from(txn_signature.as_slice()).expect("valid signature") + } else { + Signature::default() + }, pubkey: Pubkey::try_from(account_info.pubkey.as_slice()).expect("valid pubkey"), executable: account_info.executable, lamports: account_info.lamports, @@ -138,7 +139,7 @@ impl From<(SubscribeUpdateTransaction, Option)> for TransactionPretty let transaction_index = tx.index; Self { slot, - transaction_index: Some(transaction_index), // 提取交易索引 + transaction_index: Some(transaction_index), // 提取交易索引 block_time, block_hash: "".to_string(), signature: Signature::try_from(tx.signature.as_slice()).expect("valid signature"), diff --git a/src/streaming/mod.rs b/src/streaming/mod.rs index 9aed6b5..8747e2a 100755 --- a/src/streaming/mod.rs +++ b/src/streaming/mod.rs @@ -4,8 +4,8 @@ pub mod grpc; pub mod shred; pub mod shred_stream; pub mod yellowstone_grpc; -// pub mod yellowstone_sub_system; +pub mod yellowstone_sub_system; pub use shred::ShredStreamGrpc; pub use yellowstone_grpc::YellowstoneGrpc; -// pub use yellowstone_sub_system::{SystemEvent, TransferInfo}; +pub use yellowstone_sub_system::{SystemEvent, TransferInfo}; diff --git a/src/streaming/shred/connection.rs b/src/streaming/shred/connection.rs index 90bc64a..44e7e4a 100644 --- a/src/streaming/shred/connection.rs +++ b/src/streaming/shred/connection.rs @@ -29,10 +29,8 @@ impl ShredStreamGrpc { pub async fn new_with_config(endpoint: String, config: StreamClientConfig) -> AnyResult { let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?; let metrics = Arc::new(RwLock::new(PerformanceMetrics::new())); - let config_arc = Arc::new(config.clone()); - let metrics_manager = - MetricsManager::new(metrics.clone(), config_arc, "ShredStream".to_string()); + let metrics_manager = MetricsManager::new(config.enable_metrics, "ShredStream".to_string()); Ok(Self { shredstream_client: Arc::new(shredstream_client), diff --git a/src/streaming/shred_stream.rs b/src/streaming/shred_stream.rs index 860429a..79cda1b 100755 --- a/src/streaming/shred_stream.rs +++ b/src/streaming/shred_stream.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use futures::StreamExt; use solana_sdk::pubkey::Pubkey; @@ -39,8 +41,9 @@ impl ShredStreamGrpc { event_processor.set_protocols_and_event_type_filter( protocols, event_type_filter, - self.config.backpressure.strategy, + self.config.backpressure.clone(), self.config.batch.clone(), + Some(Arc::new(callback)), ); // 启动流处理 @@ -61,7 +64,6 @@ impl ShredStreamGrpc { .process_shred_transaction_immediate( transaction_with_slot, bot_wallet, - &callback, ) .await { @@ -82,11 +84,7 @@ impl ShredStreamGrpc { }); // 保存订阅句柄 - let subscription_handle = SubscriptionHandle::new( - stream_task, - event_processor.get_event_handle(), - metrics_handle, - ); + let subscription_handle = SubscriptionHandle::new(stream_task, None, metrics_handle); let mut handle_guard = self.subscription_handle.lock().await; *handle_guard = Some(subscription_handle); diff --git a/src/streaming/yellowstone_grpc.rs b/src/streaming/yellowstone_grpc.rs index 58075cd..db928d9 100644 --- a/src/streaming/yellowstone_grpc.rs +++ b/src/streaming/yellowstone_grpc.rs @@ -51,12 +51,14 @@ impl YellowstoneGrpc { ) -> AnyResult { let _ = rustls::crypto::ring::default_provider().install_default().ok(); let metrics = Arc::new(RwLock::new(PerformanceMetrics::new())); - let config_arc = Arc::new(config.clone()); let subscription_manager = SubscriptionManager::new(endpoint.clone(), x_token.clone(), config.clone()); - let metrics_manager = - MetricsManager::new(metrics.clone(), config_arc.clone(), "YellowstoneGrpc".to_string()); + let metrics_manager = MetricsManager::new_with_metrics( + metrics.clone(), + config.enable_metrics, + "YellowstoneGrpc".to_string(), + ); let event_processor = EventProcessor::new(metrics_manager.clone(), config.clone()); Ok(Self { @@ -179,8 +181,9 @@ impl YellowstoneGrpc { event_processor.set_protocols_and_event_type_filter( protocols, event_type_filter, - self.config.backpressure.strategy, + self.config.backpressure.clone(), self.config.batch.clone(), + Some(Arc::new(callback)), ); let stream_handle = tokio::spawn(async move { while let Some(message) = stream.next().await { @@ -190,7 +193,6 @@ impl YellowstoneGrpc { msg, &mut subscribe_tx, event_processor.clone(), - &callback, bot_wallet, ) .await @@ -208,11 +210,7 @@ impl YellowstoneGrpc { }); // 保存订阅句柄 - let subscription_handle = SubscriptionHandle::new( - stream_handle, - self.event_processor.get_event_handle(), - metrics_handle, - ); + let subscription_handle = SubscriptionHandle::new(stream_handle, None, metrics_handle); let mut handle_guard = self.subscription_handle.lock().await; *handle_guard = Some(subscription_handle); diff --git a/src/streaming/yellowstone_sub_system.rs b/src/streaming/yellowstone_sub_system.rs index 4d19f95..bccd417 100755 --- a/src/streaming/yellowstone_sub_system.rs +++ b/src/streaming/yellowstone_sub_system.rs @@ -1,19 +1,17 @@ use crate::{ common::AnyResult, streaming::{ - grpc::{BackpressureStrategy, EventPretty, StreamHandler}, + grpc::{EventPretty, StreamHandler}, yellowstone_grpc::YellowstoneGrpc, }, }; -use futures::{channel::mpsc, StreamExt}; +use futures::StreamExt; use log::error; use solana_program::pubkey; use solana_sdk::{pubkey::Pubkey, transaction::VersionedTransaction}; use solana_transaction_status::TransactionWithStatusMeta; const SYSTEM_PROGRAM_ID: Pubkey = pubkey!("11111111111111111111111111111111"); -// 根据实际并发量调整通道大小，避免背压 -const CHANNEL_SIZE: usize = 50000; // 增加到 50000 #[derive(Debug)] pub enum SystemEvent { @@ -51,7 +49,6 @@ impl YellowstoneGrpc { .subscription_manager .subscribe_with_request(transactions, None, None, None) .await?; - let (mut tx, mut rx) = mpsc::channel::(CHANNEL_SIZE); let callback = Box::new(callback); @@ -59,16 +56,17 @@ impl YellowstoneGrpc { while let Some(message) = stream.next().await { match message { Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut tx, - &mut subscribe_tx, - BackpressureStrategy::Block, - ) - .await + if let Ok(event_pretty) = + StreamHandler::handle_stream_system_message(msg, &mut subscribe_tx) + .await { - error!("Error handling message: {e:?}"); - break; + if let Some(event_pretty) = event_pretty { + if let Err(e) = + Self::process_system_transaction(event_pretty, &*callback).await + { + error!("Error processing transaction: {e:?}"); + } + } } } Err(error) => { @@ -78,12 +76,6 @@ impl YellowstoneGrpc { } } }); - - while let Some(event_pretty) = rx.next().await { - if let Err(e) = Self::process_system_transaction(event_pretty, &*callback).await { - error!("Error processing transaction: {e:?}"); - } - } Ok(()) } From 218abd3aa478689c3b3edc66512f8f6bd66813f4 Mon Sep 17 00:00:00 2001 From: ysq Date: Thu, 28 Aug 2025 14:49:56 +0800 Subject: [PATCH 5/7] perf: Major event processing system refactor for improved performance --- src/lib.rs | 4 +- src/streaming/common/event_processor.rs | 10 +- src/streaming/event_parser/common/mod.rs | 4 - src/streaming/event_parser/common/types.rs | 8 +- src/streaming/event_parser/common/utils.rs | 9 -- src/streaming/event_parser/core/macros.rs | 97 ++++++++++++++++ src/streaming/event_parser/core/mod.rs | 1 + src/streaming/event_parser/core/traits.rs | 66 ++++++++--- .../protocols/block/block_meta_event.rs | 1 + .../event_parser/protocols/bonk/events.rs | 8 +- .../event_parser/protocols/bonk/parser.rs | 89 +++------------ .../event_parser/protocols/mutil/parser.rs | 78 ++----------- .../event_parser/protocols/pumpfun/events.rs | 12 +- .../event_parser/protocols/pumpfun/parser.rs | 83 ++------------ .../event_parser/protocols/pumpswap/events.rs | 20 +++- .../event_parser/protocols/pumpswap/parser.rs | 85 +++----------- .../protocols/raydium_amm_v4/parser.rs | 93 +++------------ .../protocols/raydium_clmm/parser.rs | 107 ++++-------------- .../protocols/raydium_cpmm/parser.rs | 91 +++------------ 19 files changed, 292 insertions(+), 574 deletions(-) create mode 100644 src/streaming/event_parser/core/macros.rs diff --git a/src/lib.rs b/src/lib.rs index 8416f5b..2e79b9d 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,3 @@ -pub mod streaming; +pub mod common; pub mod protos; -pub mod common; \ No newline at end of file +pub mod streaming; diff --git a/src/streaming/common/event_processor.rs b/src/streaming/common/event_processor.rs index a2364fe..e7ffe13 100644 --- a/src/streaming/common/event_processor.rs +++ b/src/streaming/common/event_processor.rs @@ -115,21 +115,20 @@ impl EventProcessor { transaction_pretty.block_time, transaction_pretty.program_received_time_us, bot_wallet, + transaction_pretty.transaction_index, ) .await .unwrap_or_else(|_e| vec![]); - let mut max_time_consuming_us = 0; + let mut all_time_consuming_us = 0; let event_count = all_events.len(); // 为所有事件设置交易索引 for mut event in all_events { - event.set_transaction_index(transaction_pretty.transaction_index); event.set_program_handle_time_consuming_us( chrono::Utc::now().timestamp_micros() - event.program_received_time_us(), ); - max_time_consuming_us = - max_time_consuming_us.max(event.program_handle_time_consuming_us()); + all_time_consuming_us += event.program_handle_time_consuming_us(); self.invoke_callback(event); } @@ -137,7 +136,7 @@ impl EventProcessor { self.update_metrics( MetricsEventType::Transaction, event_count as u64, - max_time_consuming_us as f64, + all_time_consuming_us as f64, Some(signature), ); } @@ -199,6 +198,7 @@ impl EventProcessor { None, program_received_time_us, bot_wallet, + None, ) .await .unwrap_or_else(|_e| vec![]); diff --git a/src/streaming/event_parser/common/mod.rs b/src/streaming/event_parser/common/mod.rs index 85c2d90..c74b304 100755 --- a/src/streaming/event_parser/common/mod.rs +++ b/src/streaming/event_parser/common/mod.rs @@ -76,10 +76,6 @@ macro_rules! impl_unified_event { fn transaction_index(&self) -> Option { self.metadata.transaction_index } - - fn set_transaction_index(&mut self, transaction_index: Option) { - self.metadata.set_transaction_index(transaction_index); - } } }; } diff --git a/src/streaming/event_parser/common/types.rs b/src/streaming/event_parser/common/types.rs index 2bc74ba..60c1e1a 100755 --- a/src/streaming/event_parser/common/types.rs +++ b/src/streaming/event_parser/common/types.rs @@ -323,12 +323,12 @@ impl EventMetadata { instruction_outer_index: i64, instruction_inner_index: Option, program_received_time_us: i64, + transaction_index: Option, ) -> Self { Self { id, signature, slot, - transaction_index: None, // 默认为None，后续设置 block_time, block_time_ms, program_received_time_us, @@ -339,6 +339,7 @@ impl EventMetadata { swap_data: None, instruction_outer_index, instruction_inner_index, + transaction_index, } } @@ -350,11 +351,6 @@ impl EventMetadata { self.swap_data = Some(swap_data); } - /// 设置交易索引 - pub fn set_transaction_index(&mut self, transaction_index: Option) { - self.transaction_index = transaction_index; - } - /// Recycle EventMetadata to object pool pub fn recycle(self) { EVENT_METADATA_POOL.release(self); diff --git a/src/streaming/event_parser/common/utils.rs b/src/streaming/event_parser/common/utils.rs index 80458d4..56ec1a5 100755 --- a/src/streaming/event_parser/common/utils.rs +++ b/src/streaming/event_parser/common/utils.rs @@ -13,15 +13,6 @@ pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8] Some((&data[..length], &data[length..])) } -/// 检查鉴别器是否匹配 - 优化版本 -pub fn discriminator_matches(data: &str, expected: &str) -> bool { - if data.len() < expected.len() { - return false; - } - // 使用字节比较而不是字符串比较，更高效 - data.as_bytes().starts_with(expected.as_bytes()) -} - /// 从日志中提取程序数据 pub fn extract_program_data(log: &str) -> Option<&str> { const PROGRAM_DATA_PREFIX: &str = "Program data: "; diff --git a/src/streaming/event_parser/core/macros.rs b/src/streaming/event_parser/core/macros.rs new file mode 100644 index 0000000..9ef239e --- /dev/null +++ b/src/streaming/event_parser/core/macros.rs @@ -0,0 +1,97 @@ +/// Macro to generate boilerplate EventParser implementation for protocol parsers +/// +/// This macro eliminates the repetitive code where each parser simply delegates +/// all EventParser trait methods to its inner GenericEventParser. +/// +/// Usage: +/// ```rust +/// impl_event_parser_delegate!(MyEventParser); +/// ``` +/// +/// This will generate the complete EventParser implementation that delegates +/// all methods to `self.inner`. +#[macro_export] +macro_rules! impl_event_parser_delegate { + ($parser_type:ty) => { + #[async_trait::async_trait] + impl $crate::streaming::event_parser::core::traits::EventParser for $parser_type { + fn inner_instruction_configs( + &self, + ) -> std::collections::HashMap< + Vec, + Vec<$crate::streaming::event_parser::core::traits::GenericEventParseConfig>, + > { + self.inner.inner_instruction_configs() + } + + fn instruction_configs( + &self, + ) -> std::collections::HashMap< + Vec, + Vec<$crate::streaming::event_parser::core::traits::GenericEventParseConfig>, + > { + self.inner.instruction_configs() + } + + fn parse_events_from_inner_instruction( + &self, + inner_instruction: &solana_sdk::instruction::CompiledInstruction, + signature: solana_sdk::signature::Signature, + slot: u64, + block_time: Option, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + bot_wallet: Option, + transaction_index: Option, + ) -> Vec> { + self.inner.parse_events_from_inner_instruction( + inner_instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + bot_wallet, + transaction_index, + ) + } + + fn parse_events_from_instruction( + &self, + instruction: &solana_sdk::instruction::CompiledInstruction, + accounts: &[solana_sdk::pubkey::Pubkey], + signature: solana_sdk::signature::Signature, + slot: u64, + block_time: Option, + program_received_time_us: i64, + outer_index: i64, + inner_index: Option, + bot_wallet: Option, + transaction_index: Option, + ) -> Vec> { + self.inner.parse_events_from_instruction( + instruction, + accounts, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + bot_wallet, + transaction_index, + ) + } + + fn should_handle(&self, program_id: &solana_sdk::pubkey::Pubkey) -> bool { + self.inner.should_handle(program_id) + } + + fn supported_program_ids(&self) -> Vec { + self.inner.supported_program_ids() + } + } + }; +} diff --git a/src/streaming/event_parser/core/mod.rs b/src/streaming/event_parser/core/mod.rs index 27d61db..cbe0602 100755 --- a/src/streaming/event_parser/core/mod.rs +++ b/src/streaming/event_parser/core/mod.rs @@ -1,4 +1,5 @@ pub mod common_event_parser; pub mod traits; pub mod account_event_parser; +pub mod macros; pub use traits::{EventParser, UnifiedEvent}; diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index f9c7f4c..b37da1a 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -68,16 +68,13 @@ pub trait UnifiedEvent: Debug + Send + Sync { /// Get transaction index in slot fn transaction_index(&self) -> Option; - - /// Set transaction index in slot - fn set_transaction_index(&mut self, transaction_index: Option); } /// 事件解析器trait - 定义了事件解析的核心方法 #[async_trait::async_trait] pub trait EventParser: Send + Sync { /// 获取内联指令解析配置 - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec>; + fn inner_instruction_configs(&self) -> HashMap, Vec>; /// 获取指令解析配置 fn instruction_configs(&self) -> HashMap, Vec>; /// 从内联指令中解析事件数据 @@ -91,6 +88,8 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, outer_index: i64, inner_index: Option, + bot_wallet: Option, + transaction_index: Option, ) -> Vec>; /// 从指令中解析事件数据 @@ -105,6 +104,8 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, outer_index: i64, inner_index: Option, + bot_wallet: Option, + transaction_index: Option, ) -> Vec>; /// 从VersionedTransaction中解析指令事件的通用方法 @@ -118,6 +119,8 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, accounts: &[Pubkey], inner_instructions: &[InnerInstructions], + bot_wallet: Option, + transaction_index: Option, ) -> Result>> { // 预分配容量，避免动态扩容 let mut instruction_events = Vec::with_capacity(16); @@ -149,6 +152,8 @@ pub trait EventParser: Send + Sync { program_received_time_us, index as i64, None, + bot_wallet, + Some(index as u64), ) .await { @@ -188,6 +193,7 @@ pub trait EventParser: Send + Sync { block_time: Option, program_received_time_us: i64, bot_wallet: Option, + transaction_index: Option, ) -> Result>> { let accounts: Vec = versioned_tx.message.static_account_keys().to_vec(); let events = self @@ -199,6 +205,8 @@ pub trait EventParser: Send + Sync { program_received_time_us, &accounts, &[], + bot_wallet, + transaction_index, ) .await .unwrap_or_else(|_e| vec![]); @@ -213,6 +221,7 @@ pub trait EventParser: Send + Sync { block_time: Option, program_received_time_us: i64, bot_wallet: Option, + transaction_index: Option, ) -> Result>> { let versioned_tx = tx.get_transaction(); let meta = tx.get_status_meta(); @@ -254,6 +263,8 @@ pub trait EventParser: Send + Sync { program_received_time_us, &accounts_for_task1, &inner_instructions_for_task1, + bot_wallet, + transaction_index, ) .await .unwrap_or_else(|_e| vec![]) @@ -305,6 +316,8 @@ pub trait EventParser: Send + Sync { *program_received_time_us, *outer_index, *inner_index, + bot_wallet, + transaction_index, ) .await { @@ -348,6 +361,8 @@ pub trait EventParser: Send + Sync { *program_received_time_us, *outer_index, *inner_index, + bot_wallet, + transaction_index, ) .await { @@ -496,6 +511,8 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, outer_index: i64, inner_index: Option, + bot_wallet: Option, + transaction_index: Option, ) -> Result>> { let slot = slot.unwrap_or(0); let events = self.parse_events_from_inner_instruction( @@ -506,6 +523,8 @@ pub trait EventParser: Send + Sync { program_received_time_us, outer_index, inner_index, + bot_wallet, + transaction_index, ); Ok(events) } @@ -521,6 +540,8 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, outer_index: i64, inner_index: Option, + bot_wallet: Option, + transaction_index: Option, ) -> Result>> { let slot = slot.unwrap_or(0); let events = self.parse_events_from_instruction( @@ -532,6 +553,8 @@ pub trait EventParser: Send + Sync { program_received_time_us, outer_index, inner_index, + bot_wallet, + transaction_index, ); Ok(events) } @@ -555,7 +578,7 @@ impl Clone for Box { pub struct GenericEventParseConfig { pub program_id: Pubkey, pub protocol_type: ProtocolType, - pub inner_instruction_discriminator: &'static str, + pub inner_instruction_discriminator: &'static [u8], pub instruction_discriminator: &'static [u8], pub event_type: EventType, pub inner_instruction_parser: Option, @@ -573,7 +596,7 @@ pub type InstructionEventParser = /// 通用事件解析器基类 pub struct GenericEventParser { pub program_ids: Vec, - pub inner_instruction_configs: HashMap<&'static str, Vec>, + pub inner_instruction_configs: HashMap, Vec>, pub instruction_configs: HashMap, Vec>, } @@ -585,10 +608,12 @@ impl GenericEventParser { let mut instruction_configs = HashMap::with_capacity(configs.len()); for config in configs { - inner_instruction_configs - .entry(config.inner_instruction_discriminator) - .or_insert_with(Vec::new) - .push(config.clone()); + if config.inner_instruction_discriminator.len() > 0 { + inner_instruction_configs + .entry(config.inner_instruction_discriminator.to_vec()) + .or_insert_with(Vec::new) + .push(config.clone()); + } instruction_configs .entry(config.instruction_discriminator.to_vec()) .or_insert_with(Vec::new) @@ -610,6 +635,7 @@ impl GenericEventParser { program_received_time_us: i64, outer_index: i64, inner_index: Option, + transaction_index: Option, ) -> Option> { if let Some(parser) = config.inner_instruction_parser { let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); @@ -626,6 +652,7 @@ impl GenericEventParser { outer_index, inner_index, program_received_time_us, + transaction_index, ); parser(data, metadata) } else { @@ -646,6 +673,7 @@ impl GenericEventParser { program_received_time_us: i64, outer_index: i64, inner_index: Option, + transaction_index: Option, ) -> Option> { if let Some(parser) = config.instruction_parser { let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); @@ -662,6 +690,7 @@ impl GenericEventParser { outer_index, inner_index, program_received_time_us, + transaction_index, ); parser(data, account_pubkeys, metadata) } else { @@ -672,7 +701,7 @@ impl GenericEventParser { #[async_trait::async_trait] impl EventParser for GenericEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { + fn inner_instruction_configs(&self) -> HashMap, Vec> { // 返回引用而非克隆，减少内存分配 self.inner_instruction_configs.clone() } @@ -691,17 +720,16 @@ impl EventParser for GenericEventParser { program_received_time_us: i64, outer_index: i64, inner_index: Option, + bot_wallet: Option, + transaction_index: Option, ) -> Vec> { - let inner_instruction_data_decoded = inner_instruction.data.clone(); - if inner_instruction_data_decoded.len() < 16 { + if inner_instruction.data.len() < 16 { return Vec::new(); } - let inner_instruction_data_decoded_str = - format!("0x{}", hex::encode(&inner_instruction_data_decoded)); - let data = &inner_instruction_data_decoded[16..]; + let data = &inner_instruction.data[16..]; let mut events = Vec::new(); for (disc, configs) in &self.inner_instruction_configs { - if discriminator_matches(&inner_instruction_data_decoded_str, disc) { + if data == disc { for config in configs { if let Some(event) = self.parse_inner_instruction_event( config, @@ -712,6 +740,7 @@ impl EventParser for GenericEventParser { program_received_time_us, outer_index, inner_index, + transaction_index, ) { events.push(event); } @@ -733,6 +762,8 @@ impl EventParser for GenericEventParser { program_received_time_us: i64, outer_index: i64, inner_index: Option, + bot_wallet: Option, + transaction_index: Option, ) -> Vec> { let program_id = accounts[instruction.program_id_index as usize]; if !self.should_handle(&program_id) { @@ -767,6 +798,7 @@ impl EventParser for GenericEventParser { program_received_time_us, outer_index, inner_index, + transaction_index, ) { events.push(event); } diff --git a/src/streaming/event_parser/protocols/block/block_meta_event.rs b/src/streaming/event_parser/protocols/block/block_meta_event.rs index 2f99ba1..d204eef 100644 --- a/src/streaming/event_parser/protocols/block/block_meta_event.rs +++ b/src/streaming/event_parser/protocols/block/block_meta_event.rs @@ -31,6 +31,7 @@ impl BlockMetaEvent { 0, None, program_received_time_us, + None, ); Self { metadata, slot, block_hash } } diff --git a/src/streaming/event_parser/protocols/bonk/events.rs b/src/streaming/event_parser/protocols/bonk/events.rs index e2a2d50..cb450ae 100755 --- a/src/streaming/event_parser/protocols/bonk/events.rs +++ b/src/streaming/event_parser/protocols/bonk/events.rs @@ -314,8 +314,12 @@ impl_unified_event!(BonkPlatformConfigAccountEvent,); /// Event discriminator constants pub mod discriminators { // Event discriminators - pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; - pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae"; + // pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; + pub const TRADE_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238]; + // pub const POOL_CREATE_EVENT: &str = "0xe445a52e51cb9a1d97d7e20976a173ae"; + pub const POOL_CREATE_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 151, 215, 226, 9, 118, 161, 115, 174]; // Instruction discriminators pub const BUY_EXACT_IN: &[u8] = &[250, 234, 13, 123, 213, 156, 19, 236]; diff --git a/src/streaming/event_parser/protocols/bonk/parser.rs b/src/streaming/event_parser/protocols/bonk/parser.rs index bc28e85..0c08216 100755 --- a/src/streaming/event_parser/protocols/bonk/parser.rs +++ b/src/streaming/event_parser/protocols/bonk/parser.rs @@ -1,16 +1,16 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; - -use crate::streaming::event_parser::{ - common::{utils::*, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::bonk::{ - bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, - BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, - ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, - VestingParams, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{utils::*, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::bonk::{ + bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, + AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, + BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, + TradeDirection, VestingParams, + }, }, }; @@ -90,7 +90,7 @@ impl BonkEventParser { GenericEventParseConfig { program_id: BONK_PROGRAM_ID, protocol_type: ProtocolType::Bonk, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::MIGRATE_TO_AMM, event_type: EventType::BonkMigrateToAmm, inner_instruction_parser: None, @@ -99,7 +99,7 @@ impl BonkEventParser { GenericEventParseConfig { program_id: BONK_PROGRAM_ID, protocol_type: ProtocolType::Bonk, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::MIGRATE_TO_CP_SWAP, event_type: EventType::BonkMigrateToCpswap, inner_instruction_parser: None, @@ -603,63 +603,4 @@ impl BonkEventParser { } } -#[async_trait::async_trait] -impl EventParser for BonkEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(BonkEventParser); diff --git a/src/streaming/event_parser/protocols/mutil/parser.rs b/src/streaming/event_parser/protocols/mutil/parser.rs index b04da73..c8b58a0 100755 --- a/src/streaming/event_parser/protocols/mutil/parser.rs +++ b/src/streaming/event_parser/protocols/mutil/parser.rs @@ -1,13 +1,10 @@ -use std::collections::HashMap; - -use prost_types::Timestamp; -use solana_sdk::signature::Signature; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey}; - -use crate::streaming::event_parser::common::filter::EventTypeFilter; -use crate::streaming::event_parser::{ - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - EventParserFactory, Protocol, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::filter::EventTypeFilter, + core::traits::{GenericEventParseConfig, GenericEventParser}, + EventParserFactory, Protocol, + }, }; pub struct MutilEventParser { @@ -64,63 +61,4 @@ impl MutilEventParser { } } -#[async_trait::async_trait] -impl EventParser for MutilEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(MutilEventParser); diff --git a/src/streaming/event_parser/protocols/pumpfun/events.rs b/src/streaming/event_parser/protocols/pumpfun/events.rs index 89b02d7..5e06a84 100755 --- a/src/streaming/event_parser/protocols/pumpfun/events.rs +++ b/src/streaming/event_parser/protocols/pumpfun/events.rs @@ -255,9 +255,15 @@ impl_unified_event!(PumpFunGlobalAccountEvent,); /// 事件鉴别器常量 pub mod discriminators { // 事件鉴别器 - pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376"; - pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; - pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94"; + // pub const CREATE_TOKEN_EVENT: &str = "0xe445a52e51cb9a1d1b72a94ddeeb6376"; + pub const CREATE_TOKEN_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 27, 114, 169, 77, 222, 235, 99, 118]; + // pub const TRADE_EVENT: &str = "0xe445a52e51cb9a1dbddb7fd34ee661ee"; + pub const TRADE_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 189, 219, 127, 211, 78, 230, 97, 238]; + // pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &str = "0xe445a52e51cb9a1dbde95db95c94ea94"; + pub const COMPLETE_PUMP_AMM_MIGRATION_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 189, 233, 93, 185, 92, 148, 234, 148]; // 指令鉴别器 pub const CREATE_TOKEN_IX: &[u8] = &[24, 30, 200, 40, 5, 28, 7, 119]; diff --git a/src/streaming/event_parser/protocols/pumpfun/parser.rs b/src/streaming/event_parser/protocols/pumpfun/parser.rs index 2e97979..5596f38 100755 --- a/src/streaming/event_parser/protocols/pumpfun/parser.rs +++ b/src/streaming/event_parser/protocols/pumpfun/parser.rs @@ -1,15 +1,15 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; - -use crate::streaming::event_parser::{ - common::{EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::pumpfun::{ - discriminators, pumpfun_create_token_event_log_decode, pumpfun_migrate_event_log_decode, - pumpfun_trade_event_log_decode, PumpFunCreateTokenEvent, PumpFunMigrateEvent, - PumpFunTradeEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::pumpfun::{ + discriminators, pumpfun_create_token_event_log_decode, + pumpfun_migrate_event_log_decode, pumpfun_trade_event_log_decode, + PumpFunCreateTokenEvent, PumpFunMigrateEvent, PumpFunTradeEvent, + }, }, }; @@ -284,63 +284,4 @@ impl PumpFunEventParser { } } -#[async_trait::async_trait] -impl EventParser for PumpFunEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(PumpFunEventParser); diff --git a/src/streaming/event_parser/protocols/pumpswap/events.rs b/src/streaming/event_parser/protocols/pumpswap/events.rs index 5b27449..63a85b7 100755 --- a/src/streaming/event_parser/protocols/pumpswap/events.rs +++ b/src/streaming/event_parser/protocols/pumpswap/events.rs @@ -392,11 +392,21 @@ impl_unified_event!(PumpSwapPoolAccountEvent,); /// 事件鉴别器常量 pub mod discriminators { // 事件鉴别器 - pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777"; - pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a"; - pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774"; - pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90"; - pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0"; + // pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777"; + pub const BUY_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 103, 244, 82, 31, 44, 245, 119, 119]; + // pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a"; + pub const SELL_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 62, 47, 55, 10, 165, 3, 220, 42]; + // pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774"; + pub const CREATE_POOL_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 177, 49, 12, 210, 160, 118, 167, 116]; + // pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90"; + pub const DEPOSIT_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 120, 248, 61, 83, 31, 142, 107, 144]; + // pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0"; + pub const WITHDRAW_EVENT: &[u8] = + &[228, 69, 165, 46, 81, 203, 154, 29, 22, 9, 133, 26, 160, 44, 71, 192]; // 指令鉴别器 pub const BUY_IX: &[u8] = &[102, 6, 61, 18, 1, 218, 235, 234]; diff --git a/src/streaming/event_parser/protocols/pumpswap/parser.rs b/src/streaming/event_parser/protocols/pumpswap/parser.rs index a688c41..dc22336 100755 --- a/src/streaming/event_parser/protocols/pumpswap/parser.rs +++ b/src/streaming/event_parser/protocols/pumpswap/parser.rs @@ -1,16 +1,16 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; - -use crate::streaming::event_parser::{ - common::{read_u64_le, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::pumpswap::{ - discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode, - pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode, - pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent, - PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{read_u64_le, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::pumpswap::{ + discriminators, pump_swap_buy_event_log_decode, pump_swap_create_pool_event_log_decode, + pump_swap_deposit_event_log_decode, pump_swap_sell_event_log_decode, + pump_swap_withdraw_event_log_decode, PumpSwapBuyEvent, PumpSwapCreatePoolEvent, + PumpSwapDepositEvent, PumpSwapSellEvent, PumpSwapWithdrawEvent, + }, }, }; @@ -374,63 +374,4 @@ impl PumpSwapEventParser { } } -#[async_trait::async_trait] -impl EventParser for PumpSwapEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(PumpSwapEventParser); diff --git a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs index 288e523..6b8a7ba 100755 --- a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs @@ -1,14 +1,14 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; - -use crate::streaming::event_parser::{ - common::{read_u64_le, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::raydium_amm_v4::{ - discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event, - RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{read_u64_le, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::raydium_amm_v4::{ + discriminators, RaydiumAmmV4DepositEvent, RaydiumAmmV4Initialize2Event, + RaydiumAmmV4SwapEvent, RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent, + }, }, }; @@ -34,7 +34,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_IN, event_type: EventType::RaydiumAmmV4SwapBaseIn, inner_instruction_parser: None, @@ -43,7 +43,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_OUT, event_type: EventType::RaydiumAmmV4SwapBaseOut, inner_instruction_parser: None, @@ -52,7 +52,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::DEPOSIT, event_type: EventType::RaydiumAmmV4Deposit, inner_instruction_parser: None, @@ -61,7 +61,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::INITIALIZE2, event_type: EventType::RaydiumAmmV4Initialize2, inner_instruction_parser: None, @@ -70,7 +70,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::WITHDRAW, event_type: EventType::RaydiumAmmV4Withdraw, inner_instruction_parser: None, @@ -79,7 +79,7 @@ impl RaydiumAmmV4EventParser { GenericEventParseConfig { program_id: RAYDIUM_AMM_V4_PROGRAM_ID, protocol_type: ProtocolType::RaydiumAmmV4, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::WITHDRAW_PNL, event_type: EventType::RaydiumAmmV4WithdrawPnl, inner_instruction_parser: None, @@ -376,63 +376,4 @@ impl RaydiumAmmV4EventParser { } } -#[async_trait::async_trait] -impl EventParser for RaydiumAmmV4EventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(RaydiumAmmV4EventParser); diff --git a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs index 16fab61..7f56274 100755 --- a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs @@ -1,19 +1,19 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; - -use crate::streaming::event_parser::{ - common::{ - read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata, - EventType, ProtocolType, - }, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::raydium_clmm::{ - discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent, - RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event, - RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent, - RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{ + read_i32_le, read_option_bool, read_u128_le, read_u64_le, read_u8_le, EventMetadata, + EventType, ProtocolType, + }, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::raydium_clmm::{ + discriminators, RaydiumClmmClosePositionEvent, RaydiumClmmCreatePoolEvent, + RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLiquidityV2Event, + RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent, + RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event, + }, }, }; @@ -39,7 +39,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP, event_type: EventType::RaydiumClmmSwap, inner_instruction_parser: None, @@ -48,7 +48,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_V2, event_type: EventType::RaydiumClmmSwapV2, inner_instruction_parser: None, @@ -57,7 +57,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::CLOSE_POSITION, event_type: EventType::RaydiumClmmClosePosition, inner_instruction_parser: None, @@ -66,7 +66,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::DECREASE_LIQUIDITY_V2, event_type: EventType::RaydiumClmmDecreaseLiquidityV2, inner_instruction_parser: None, @@ -75,7 +75,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::CREATE_POOL, event_type: EventType::RaydiumClmmCreatePool, inner_instruction_parser: None, @@ -84,7 +84,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::INCREASE_LIQUIDITY_V2, event_type: EventType::RaydiumClmmIncreaseLiquidityV2, inner_instruction_parser: None, @@ -93,7 +93,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::OPEN_POSITION_WITH_TOKEN_22_NFT, event_type: EventType::RaydiumClmmOpenPositionWithToken22Nft, inner_instruction_parser: None, @@ -102,7 +102,7 @@ impl RaydiumClmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CLMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumClmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::OPEN_POSITION_V2, event_type: EventType::RaydiumClmmOpenPositionV2, inner_instruction_parser: None, @@ -417,63 +417,4 @@ impl RaydiumClmmEventParser { } } -#[async_trait::async_trait] -impl EventParser for RaydiumClmmEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(RaydiumClmmEventParser); diff --git a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs index 288bf60..f7d2e2e 100755 --- a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs @@ -1,14 +1,14 @@ -use std::collections::HashMap; +use solana_sdk::pubkey::Pubkey; -use prost_types::Timestamp; -use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; - -use crate::streaming::event_parser::{ - common::{read_u64_le, EventMetadata, EventType, ProtocolType}, - core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent}, - protocols::raydium_cpmm::{ - discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, RaydiumCpmmSwapEvent, - RaydiumCpmmWithdrawEvent, +use crate::{ + impl_event_parser_delegate, + streaming::event_parser::{ + common::{read_u64_le, EventMetadata, EventType, ProtocolType}, + core::traits::{GenericEventParseConfig, GenericEventParser, UnifiedEvent}, + protocols::raydium_cpmm::{ + discriminators, RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, + RaydiumCpmmSwapEvent, RaydiumCpmmWithdrawEvent, + }, }, }; @@ -34,7 +34,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_IN, event_type: EventType::RaydiumCpmmSwapBaseInput, inner_instruction_parser: None, @@ -43,7 +43,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::SWAP_BASE_OUT, event_type: EventType::RaydiumCpmmSwapBaseOutput, inner_instruction_parser: None, @@ -52,7 +52,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::DEPOSIT, event_type: EventType::RaydiumCpmmDeposit, inner_instruction_parser: None, @@ -61,7 +61,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::INITIALIZE, event_type: EventType::RaydiumCpmmInitialize, inner_instruction_parser: None, @@ -70,7 +70,7 @@ impl RaydiumCpmmEventParser { GenericEventParseConfig { program_id: RAYDIUM_CPMM_PROGRAM_ID, protocol_type: ProtocolType::RaydiumCpmm, - inner_instruction_discriminator: "", + inner_instruction_discriminator: &[], instruction_discriminator: discriminators::WITHDRAW, event_type: EventType::RaydiumCpmmWithdraw, inner_instruction_parser: None, @@ -267,63 +267,4 @@ impl RaydiumCpmmEventParser { } } -#[async_trait::async_trait] -impl EventParser for RaydiumCpmmEventParser { - fn inner_instruction_configs(&self) -> HashMap<&'static str, Vec> { - self.inner.inner_instruction_configs() - } - fn instruction_configs(&self) -> HashMap, Vec> { - self.inner.instruction_configs() - } - fn parse_events_from_inner_instruction( - &self, - inner_instruction: &CompiledInstruction, - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_inner_instruction( - inner_instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn parse_events_from_instruction( - &self, - instruction: &CompiledInstruction, - accounts: &[Pubkey], - signature: Signature, - slot: u64, - block_time: Option, - program_received_time_us: i64, - outer_index: i64, - inner_index: Option, - ) -> Vec> { - self.inner.parse_events_from_instruction( - instruction, - accounts, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - ) - } - - fn should_handle(&self, program_id: &Pubkey) -> bool { - self.inner.should_handle(program_id) - } - - fn supported_program_ids(&self) -> Vec { - self.inner.supported_program_ids() - } -} +impl_event_parser_delegate!(RaydiumCpmmEventParser); From b535bdf0521984b2e6081f97b4ab3f3beba608a6 Mon Sep 17 00:00:00 2001 From: ysq Date: Fri, 29 Aug 2025 14:59:16 +0800 Subject: [PATCH 6/7] perf: Major event processing system refactor for improved performance --- Cargo.toml | 1 + README.md | 87 ++- README_CN.md | 87 ++- examples/parse_tx_events.rs | 123 +++ examples/parse_tx_events.rs.bak | 196 ----- src/main.rs | 26 +- src/streaming/common/config.rs | 110 +-- src/streaming/common/constants.rs | 2 - src/streaming/common/event_processor.rs | 292 +++++-- src/streaming/common/metrics.rs | 257 +++++- .../event_parser/core/global_state.rs | 141 ++++ src/streaming/event_parser/core/macros.rs | 23 +- src/streaming/event_parser/core/mod.rs | 1 + src/streaming/event_parser/core/traits.rs | 732 +++++++++--------- .../event_parser/protocols/mutil/parser.rs | 18 - src/streaming/grpc/mod.rs | 6 +- src/streaming/grpc/stream_handler.rs | 100 --- src/streaming/shred/connection.rs | 25 +- src/streaming/shred/mod.rs | 4 +- src/streaming/shred/types.rs | 14 +- src/streaming/shred_stream.rs | 31 +- src/streaming/yellowstone_grpc.rs | 126 ++- src/streaming/yellowstone_sub_system.rs | 49 +- 23 files changed, 1557 insertions(+), 894 deletions(-) create mode 100644 examples/parse_tx_events.rs delete mode 100644 examples/parse_tx_events.rs.bak create mode 100644 src/streaming/event_parser/core/global_state.rs delete mode 100644 src/streaming/grpc/stream_handler.rs diff --git a/Cargo.toml b/Cargo.toml index 040d079..d3ed2ce 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,3 +65,4 @@ maplit = "1.0.2" env_logger = "0.11.8" crossbeam = "0.8.4" crossbeam-queue = "0.3.12" +parking_lot = "0.12.1" diff --git a/README.md b/README.md index cc73cf0..d856b6f 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ A lightweight Rust library for real-time event streaming from Solana DEX trading 11. **Performance Monitoring**: Built-in performance metrics monitoring, including event processing speed, etc. 12. **Memory Optimization**: Object pooling and caching mechanisms to reduce memory allocations 13. **Flexible Configuration System**: Support for custom batch sizes, backpressure strategies, channel sizes, and other parameters -14. **Preset Configurations**: Provides high-performance, low-latency, ordered processing, and other preset configurations +14. **Preset Configurations**: Provides high-throughput, low-latency, and async processing preset configurations optimized for different use cases 15. **Backpressure Handling**: Supports blocking, dropping, retrying, ordered, and other backpressure strategies 16. **Runtime Configuration Updates**: Supports dynamic configuration parameter updates at runtime 17. **Full Function Performance Monitoring**: All subscribe_events functions support performance monitoring, automatically collecting and reporting performance metrics @@ -55,6 +55,91 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" } solana-streamer-sdk = "0.3.10" ``` +## Configuration System + +### Preset Configurations + +The library provides three preset configurations optimized for different use cases: + +#### 1. High Throughput Configuration (`high_throughput()`) + +Optimized for high-concurrency scenarios, prioritizing throughput over latency: + +```rust +let config = StreamClientConfig::high_throughput(); +// Or use convenience methods +let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?; +let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?; +``` + +**Features:** +- **Backpressure Strategy**: Drop - drops messages during high load to avoid blocking +- **Buffer Size**: 5,000 permits to handle burst traffic +- **Use Case**: Scenarios where you need to process large volumes of data and can tolerate occasional message drops during peak loads + +#### 2. Low Latency Configuration (`low_latency()`) + +Optimized for real-time scenarios, prioritizing latency over throughput: + +```rust +let config = StreamClientConfig::low_latency(); +// Or use convenience methods +let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?; +let shred = ShredStreamGrpc::new_low_latency(endpoint).await?; +``` + +**Features:** +- **Backpressure Strategy**: Block - ensures no data loss +- **Buffer Size**: 1 permit to minimize memory usage +- **Immediate Processing**: No buffering, processes events immediately +- **Use Case**: Scenarios where every millisecond counts and you cannot afford to lose any events, such as trading applications or real-time monitoring + +#### 3. Async Processing Configuration (`async_processing()`) + +Balances throughput and reliability: + +```rust +let config = StreamClientConfig::async_processing(); +// Or use convenience methods +let grpc = YellowstoneGrpc::new_async_processing(endpoint, token)?; +let shred = ShredStreamGrpc::new_async_processing(endpoint).await?; +``` + +**Features:** +- **Backpressure Strategy**: Async - non-blocking operation +- **Buffer Size**: 5,000 permits for steady flow +- **Fire-and-forget**: Async processing semantics +- **Use Case**: Scenarios where you need sustained high throughput with eventual consistency, such as data ingestion pipelines or event streaming applications + +### Custom Configuration + +You can also create custom configurations: + +```rust +let config = StreamClientConfig { + connection: ConnectionConfig { + connect_timeout: 30, + request_timeout: 120, + max_decoding_message_size: 20 * 1024 * 1024, // 20MB + }, + backpressure: BackpressureConfig { + permits: 2000, + strategy: BackpressureStrategy::Block, + }, + enable_metrics: true, +}; +``` + +### Configuration Selection Guide + +| Scenario | Recommended Config | Reason | +|----------|-------------------|---------| +| Trading Bots | `low_latency()` | Need fastest response time, cannot lose trading signals | +| Data Analytics | `high_throughput()` | Need to process large amounts of historical data, can tolerate some data loss | +| Event Stream Processing | `async_processing()` | Balance performance and reliability, suitable for continuous processing | +| Real-time Monitoring | `low_latency()` | Need immediate response to anomalies | +| Bulk Data Ingestion | `high_throughput()` | Prioritize overall throughput | + ## Usage Examples ### Quick Start - Parse Transaction Events diff --git a/README_CN.md b/README_CN.md index 0a63871..cf4330a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -24,7 +24,7 @@ 11. **性能监控**: 内置性能指标监控，包括事件处理速度等 12. **内存优化**: 对象池和缓存机制减少内存分配 13. **灵活配置系统**: 支持自定义批处理大小、背压策略、通道大小等参数 -14. **预设配置**: 提供高性能、低延迟、有序处理等预设配置 +14. **预设配置**: 提供高吞吐量、低延迟、异步处理等预设配置，针对不同使用场景优化 15. **背压处理**: 支持阻塞、丢弃、重试、有序等多种背压策略 16. **运行时配置更新**: 支持在运行时动态更新配置参数 17. **全函数性能监控**: 所有subscribe_events函数都支持性能监控，自动收集和报告性能指标 @@ -55,6 +55,91 @@ solana-streamer-sdk = { path = "./solana-streamer", version = "0.3.10" } solana-streamer-sdk = "0.3.10" ``` +## 配置系统 + +### 预设配置 + +库提供了三种预设配置，针对不同的使用场景进行了优化： + +#### 1. 高吞吐量配置 (`high_throughput()`) + +专为高并发场景优化，优先考虑吞吐量而非延迟： + +```rust +let config = StreamClientConfig::high_throughput(); +// 或者使用便捷方法 +let grpc = YellowstoneGrpc::new_high_throughput(endpoint, token)?; +let shred = ShredStreamGrpc::new_high_throughput(endpoint).await?; +``` + +**特性：** +- **背压策略**: Drop（丢弃策略）- 在高负载时丢弃消息以避免阻塞 +- **缓冲区大小**: 5,000 个许可证，处理突发流量 +- **适用场景**: 需要处理大量数据且可以容忍在峰值负载时偶尔丢失消息的场景 + +#### 2. 低延迟配置 (`low_latency()`) + +专为实时场景优化，优先考虑延迟而非吞吐量： + +```rust +let config = StreamClientConfig::low_latency(); +// 或者使用便捷方法 +let grpc = YellowstoneGrpc::new_low_latency(endpoint, token)?; +let shred = ShredStreamGrpc::new_low_latency(endpoint).await?; +``` + +**特性：** +- **背压策略**: Block（阻塞策略）- 确保不丢失任何数据 +- **缓冲区大小**: 1 个许可证，最小化内存使用 +- **立即处理**: 不进行缓冲，立即处理事件 +- **适用场景**: 每毫秒都很重要且不能丢失任何事件的场景，如交易应用或实时监控 + +#### 3. 异步处理配置 (`async_processing()`) + +在吞吐量和可靠性之间取得平衡： + +```rust +let config = StreamClientConfig::async_processing(); +// 或者使用便捷方法 +let grpc = YellowstoneGrpc::new_async_processing(endpoint, token)?; +let shred = ShredStreamGrpc::new_async_processing(endpoint).await?; +``` + +**特性：** +- **背压策略**: Async（异步策略）- 非阻塞操作 +- **缓冲区大小**: 5,000 个许可证，保持稳定流量 +- **Fire-and-forget**: 异步处理语义 +- **适用场景**: 需要持续高吞吐量且可接受最终一致性的场景，如数据摄取管道或事件流应用 + +### 自定义配置 + +您也可以创建自定义配置： + +```rust +let config = StreamClientConfig { + connection: ConnectionConfig { + connect_timeout: 30, + request_timeout: 120, + max_decoding_message_size: 20 * 1024 * 1024, // 20MB + }, + backpressure: BackpressureConfig { + permits: 2000, + strategy: BackpressureStrategy::Block, + }, + enable_metrics: true, +}; +``` + +### 配置选择指南 + +| 场景 | 推荐配置 | 原因 | +|------|----------|------| +| 交易机器人 | `low_latency()` | 需要最快响应时间，不能丢失交易信号 | +| 数据分析 | `high_throughput()` | 需要处理大量历史数据，可容忍部分数据丢失 | +| 事件流处理 | `async_processing()` | 平衡性能和可靠性，适合持续处理 | +| 实时监控 | `low_latency()` | 需要立即响应异常情况 | +| 批量数据摄取 | `high_throughput()` | 优先考虑整体吞吐量 | + ## 使用示例 ### 快速开始 - 解析交易事件 diff --git a/examples/parse_tx_events.rs b/examples/parse_tx_events.rs new file mode 100644 index 0000000..40b9a02 --- /dev/null +++ b/examples/parse_tx_events.rs @@ -0,0 +1,123 @@ +use anyhow::Result; +use solana_sdk::commitment_config::CommitmentConfig; + +use solana_streamer_sdk::streaming::event_parser::UnifiedEvent; +use solana_streamer_sdk::streaming::event_parser::{ + protocols::MutilEventParser, EventParser, Protocol, +}; +use solana_transaction_status::{InnerInstruction, InnerInstructions, UiInstruction}; + +use solana_sdk::bs58; +use solana_sdk::instruction::CompiledInstruction; +use std::str::FromStr; +use std::sync::Arc; + +/// Get transaction data based on transaction signature +#[tokio::main] +async fn main() -> Result<()> { + let signatures = vec![ + "4PsHYajH87x2zJPEGZczZtd2ksibuMCFPonC24jk5mTGZ46hzvjpzM5UZuLz9sRv79MkCBbtDqwJapGPTSkCFKoL", + ]; + // Validate signature format + let mut valid_signatures = Vec::new(); + for sig_str in &signatures { + match solana_sdk::signature::Signature::from_str(sig_str) { + Ok(_) => valid_signatures.push(*sig_str), + Err(e) => println!("Invalid signature format: {}", e), + } + } + if valid_signatures.is_empty() { + println!("No valid transaction signatures"); + return Ok(()); + } + for signature in valid_signatures { + println!("Starting transaction parsing: {}", signature); + get_single_transaction_details(signature).await?; + println!("Transaction parsing completed: {}\n", signature); + println!("Visit link to compare data: \nhttps://solscan.io/tx/{}\n", signature); + println!("--------------------------------"); + } + + Ok(()) +} + +/// Get details of a single transaction +async fn get_single_transaction_details(signature_str: &str) -> Result<()> { + use solana_sdk::signature::Signature; + use solana_transaction_status::UiTransactionEncoding; + + let signature = Signature::from_str(signature_str)?; + + // Create Solana RPC client + let rpc_url = "https://api.mainnet-beta.solana.com"; + println!("Connecting to Solana RPC: {}", rpc_url); + + let client = solana_client::nonblocking::rpc_client::RpcClient::new(rpc_url.to_string()); + + match client + .get_transaction_with_config( + &signature, + solana_client::rpc_config::RpcTransactionConfig { + encoding: Some(UiTransactionEncoding::Base64), + commitment: Some(CommitmentConfig::confirmed()), + max_supported_transaction_version: Some(0), + }, + ) + .await + { + Ok(transaction) => { + println!("Transaction signature: {}", signature_str); + println!("Block slot: {}", transaction.slot); + + if let Some(block_time) = transaction.block_time { + println!("Block time: {}", block_time); + } + + if let Some(meta) = &transaction.transaction.meta { + println!("Transaction fee: {} lamports", meta.fee); + println!("Status: {}", if meta.err.is_none() { "Success" } else { "Failed" }); + if let Some(err) = &meta.err { + println!("Error details: {:?}", err); + } + // Compute units consumed + if let solana_transaction_status::option_serializer::OptionSerializer::Some(units) = + &meta.compute_units_consumed + { + println!("Compute units consumed: {}", units); + } + // Display logs (all) + if let solana_transaction_status::option_serializer::OptionSerializer::Some(logs) = + &meta.log_messages + { + println!("Transaction logs (all {} entries):", logs.len()); + for (i, log) in logs.iter().enumerate() { + println!(" [{}] {}", i + 1, log); + } + } + } + let protocols = vec![ + Protocol::Bonk, + Protocol::RaydiumClmm, + Protocol::PumpSwap, + Protocol::PumpFun, + Protocol::RaydiumCpmm, + Protocol::RaydiumAmmV4, + ]; + let parser: Arc = Arc::new(MutilEventParser::new(protocols, None)); + parser + .parse_encoded_confirmed_transaction_with_status_meta( + signature, + transaction, + Arc::new(move |event: &Box| { + println!("{:?}\n", event); + }), + ) + .await?; + } + Err(e) => { + println!("Failed to get transaction: {}", e); + } + } + + Ok(()) +} diff --git a/examples/parse_tx_events.rs.bak b/examples/parse_tx_events.rs.bak deleted file mode 100644 index 4ce1b6b..0000000 --- a/examples/parse_tx_events.rs.bak +++ /dev/null @@ -1,196 +0,0 @@ -use anyhow::Result; -use solana_sdk::commitment_config::CommitmentConfig; -use solana_sdk::message::v0::LoadedAddresses; -use solana_streamer_sdk::streaming::event_parser::{ - protocols::MutilEventParser, EventParser, Protocol, -}; -use solana_transaction_status::{ - option_serializer::OptionSerializer, TransactionStatusMeta, TransactionWithStatusMeta, - VersionedTransactionWithStatusMeta, -}; -use std::str::FromStr; -use std::sync::Arc; - -/// Get transaction data based on transaction signature -#[tokio::main] -async fn main() -> Result<()> { - let signatures = vec![ - "5cnxDiHzUTUutMwnTCsvnMhyL9jEQsRWimmWU1gKxpQBngDaGTkou1YbGhUJhAhmTgvu49PYMmFQbbR38wdZDxJF", - ]; - // Validate signature format - let mut valid_signatures = Vec::new(); - for sig_str in &signatures { - match solana_sdk::signature::Signature::from_str(sig_str) { - Ok(_) => valid_signatures.push(*sig_str), - Err(e) => println!("Invalid signature format: {}", e), - } - } - if valid_signatures.is_empty() { - println!("No valid transaction signatures"); - return Ok(()); - } - for signature in valid_signatures { - println!("Starting transaction parsing: {}", signature); - get_single_transaction_details(signature).await?; - println!("Transaction parsing completed: {}\n", signature); - println!("Visit link to compare data: \nhttps://solscan.io/tx/{}\n", signature); - println!("--------------------------------"); - } - - Ok(()) -} - -/// Get details of a single transaction -async fn get_single_transaction_details(signature_str: &str) -> Result<()> { - use solana_sdk::signature::Signature; - use solana_transaction_status::UiTransactionEncoding; - - let signature = Signature::from_str(signature_str)?; - - // Create Solana RPC client - let rpc_url = "https://api.mainnet-beta.solana.com"; - println!("Connecting to Solana RPC: {}", rpc_url); - - let client = solana_client::nonblocking::rpc_client::RpcClient::new(rpc_url.to_string()); - - match client - .get_transaction_with_config( - &signature, - solana_client::rpc_config::RpcTransactionConfig { - encoding: Some(UiTransactionEncoding::Base64), - commitment: Some(CommitmentConfig::confirmed()), - max_supported_transaction_version: Some(0), - }, - ) - .await - { - Ok(transaction) => { - println!("Transaction signature: {}", signature_str); - println!("Block slot: {}", transaction.slot); - - if let Some(block_time) = transaction.block_time { - println!("Block time: {}", block_time); - } - - if let Some(meta) = &transaction.transaction.meta { - println!("Transaction fee: {} lamports", meta.fee); - println!("Status: {}", if meta.err.is_none() { "Success" } else { "Failed" }); - if let Some(err) = &meta.err { - println!("Error details: {:?}", err); - } - // Compute units consumed - if let solana_transaction_status::option_serializer::OptionSerializer::Some(units) = - &meta.compute_units_consumed - { - println!("Compute units consumed: {}", units); - } - // Display logs (all) - if let solana_transaction_status::option_serializer::OptionSerializer::Some(logs) = - &meta.log_messages - { - println!("Transaction logs (all {} entries):", logs.len()); - for (i, log) in logs.iter().enumerate() { - println!(" [{}] {}", i + 1, log); - } - } - } - let protocols = vec![ - Protocol::Bonk, - Protocol::RaydiumClmm, - Protocol::PumpSwap, - Protocol::PumpFun, - Protocol::RaydiumCpmm, - Protocol::RaydiumAmmV4, - ]; - let parser: Arc = Arc::new(MutilEventParser::new(protocols, None)); - let start_time = std::time::Instant::now(); - - // 从 EncodedTransaction 获取 VersionedTransaction - let versioned_tx = match transaction.transaction.transaction.decode() { - Some(tx) => tx, - None => { - println!("Failed to decode transaction"); - return Ok(()); - } - }; - - // 创建 TransactionWithStatusMeta - let tx = TransactionWithStatusMeta::Complete(VersionedTransactionWithStatusMeta { - transaction: versioned_tx, - meta: TransactionStatusMeta { - status: Ok(()), - fee: transaction.transaction.meta.as_ref().map_or(0, |m| m.fee), - pre_balances: transaction - .transaction - .meta - .as_ref() - .map_or(vec![], |m| m.pre_balances.clone()), - post_balances: transaction - .transaction - .meta - .as_ref() - .map_or(vec![], |m| m.post_balances.clone()), - inner_instructions: transaction.transaction.meta.as_ref().and_then(|m| { - if let OptionSerializer::Some(inner_instructions) = &m.inner_instructions { - // 手动将每个UiInnerInstructions转换为InnerInstructions - Some(inner_instructions.iter().map(|ui_inner| { - solana_transaction_status::InnerInstructions { - index: ui_inner.index, - instructions: ui_inner.instructions.iter().map(|ui_inst| { - solana_transaction_status::InnerInstruction { - instruction: solana_sdk::instruction::Instruction { - program_id: solana_sdk::pubkey::Pubkey::new_from_array([0; 32]), - accounts: vec![], - data: vec![], - }, - stack_height: None, - } - }).collect(), - } - }).collect()) - } else { - None - } - }), - log_messages: transaction.transaction.meta.as_ref().and_then(|m| { - if let OptionSerializer::Some(logs) = &m.log_messages { - Some(logs.clone()) - } else { - None - } - }), - pre_token_balances: None, - post_token_balances: None, - rewards: None, - loaded_addresses: LoadedAddresses::default(), - return_data: None, - compute_units_consumed: None, - cost_units: None, - }, - }); - - // TransactionWithStatusMeta - let events = parser - .parse_transaction( - tx, - &signature.to_string(), - Some(transaction.slot), - None, - chrono::Utc::now().timestamp_micros(), - None, - ) - .await - .unwrap_or_else(|_e| vec![]); - - println!("Parsing time: {:?}", start_time.elapsed()); - for event in events { - println!("{:?}\n", event); - } - } - Err(e) => { - println!("Failed to get transaction: {}", e); - } - } - - Ok(()) -} diff --git a/src/main.rs b/src/main.rs index b95723c..d6ad5c3 100755 --- a/src/main.rs +++ b/src/main.rs @@ -61,7 +61,7 @@ async fn test_grpc() -> Result<(), Box> { println!("Subscribing to Yellowstone gRPC events..."); // Create low-latency configuration - let mut config = ClientConfig::low_latency(); + let mut config: ClientConfig = ClientConfig::low_latency(); // Enable performance monitoring, has performance overhead, disabled by default config.enable_metrics = true; let grpc = YellowstoneGrpc::new_with_config( @@ -86,17 +86,13 @@ async fn test_grpc() -> Result<(), Box> { println!("Protocols to monitor: {:?}", protocols); - const SYSTEM_PROGRAM_ID: solana_sdk::pubkey::Pubkey = - solana_sdk::pubkey!("11111111111111111111111111111111"); - // Filter accounts let account_include = vec![ - SYSTEM_PROGRAM_ID.to_string(), - PUMPFUN_PROGRAM_ID.to_string(), // Listen to pumpfun program ID - PUMPSWAP_PROGRAM_ID.to_string(), // Listen to pumpswap program ID - BONK_PROGRAM_ID.to_string(), // Listen to bonk program ID - RAYDIUM_CPMM_PROGRAM_ID.to_string(), // Listen to raydium_cpmm program ID - RAYDIUM_CLMM_PROGRAM_ID.to_string(), // Listen to raydium_clmm program ID + PUMPFUN_PROGRAM_ID.to_string(), // Listen to pumpfun program ID + PUMPSWAP_PROGRAM_ID.to_string(), // Listen to pumpswap program ID + BONK_PROGRAM_ID.to_string(), // Listen to bonk program ID + RAYDIUM_CPMM_PROGRAM_ID.to_string(), // Listen to raydium_cpmm program ID + RAYDIUM_CLMM_PROGRAM_ID.to_string(), // Listen to raydium_clmm program ID RAYDIUM_AMM_V4_PROGRAM_ID.to_string(), // Listen to raydium_amm_v4 program ID ]; let account_exclude = vec![]; @@ -116,7 +112,7 @@ async fn test_grpc() -> Result<(), Box> { // No event filtering, includes all events let event_type_filter = None; // Only include PumpSwapBuy events and PumpSwapSell events - // let event_type_filter = EventTypeFilter { include: vec![EventType::PumpSwapBuy, EventType::PumpSwapSell] }; + // let event_type_filter = Some(EventTypeFilter { include: vec![EventType::PumpFunBuy] }); println!("Starting to listen for events, press Ctrl+C to stop..."); println!("Monitoring programs: {:?}", account_include); @@ -155,7 +151,7 @@ async fn test_shreds() -> Result<(), Box> { // Enable performance monitoring, has performance overhead, disabled by default config.enable_metrics = true; let shred_stream = - ShredStreamGrpc::new_with_config("http://64.130.37.195:10800".to_string(), config).await?; + ShredStreamGrpc::new_with_config("http://127.0.0.1:10800".to_string(), config).await?; let callback = create_event_callback(); let protocols = vec![ @@ -192,7 +188,11 @@ async fn test_shreds() -> Result<(), Box> { fn create_event_callback() -> impl Fn(Box) { |event: Box| { - println!("🎉 Event received! Type: {:?}, ID: {}", event.event_type(), event.id()); + println!( + "🎉 Event received! Type: {:?}, transaction_index: {:?}", + event.event_type(), + event.transaction_index() + ); match_event!(event, { // -------------------------- block meta ----------------------- BlockMetaEvent => |e: BlockMetaEvent| { diff --git a/src/streaming/common/config.rs b/src/streaming/common/config.rs index c050016..0c76687 100644 --- a/src/streaming/common/config.rs +++ b/src/streaming/common/config.rs @@ -1,14 +1,14 @@ use super::constants::*; -/// 背压处理策略 +/// Backpressure handling strategy #[derive(Debug, Clone, Copy)] pub enum BackpressureStrategy { - /// 阻塞等待（默认） + /// Block and wait (default) Block, - /// 丢弃消息 + /// Drop messages Drop, - /// 重试有限次数后丢弃 - Retry { max_attempts: usize, wait_ms: u64 }, + /// Execute asynchronously (don't wait for completion) + Async, } impl Default for BackpressureStrategy { @@ -17,50 +17,29 @@ impl Default for BackpressureStrategy { } } -/// 批处理配置 -#[derive(Debug, Clone)] -pub struct BatchConfig { - /// 批处理大小（默认：100） - pub batch_size: usize, - /// 批处理超时时间（毫秒，默认：5ms） - pub batch_timeout_ms: u64, - /// 是否启用批处理（默认：true） - pub enabled: bool, -} - -impl Default for BatchConfig { - fn default() -> Self { - Self { - batch_size: DEFAULT_BATCH_SIZE, - batch_timeout_ms: DEFAULT_BATCH_TIMEOUT_MS, - enabled: true, - } - } -} - -/// 背压配置 +/// Backpressure configuration #[derive(Debug, Clone)] pub struct BackpressureConfig { - /// 通道大小（默认：1000） - pub channel_size: usize, - /// 背压处理策略（默认：Block） + /// Channel size (default: 1000) + pub permits: usize, + /// Backpressure handling strategy (default: Block) pub strategy: BackpressureStrategy, } impl Default for BackpressureConfig { fn default() -> Self { - Self { channel_size: DEFAULT_CHANNEL_SIZE, strategy: BackpressureStrategy::default() } + Self { permits: 1, strategy: BackpressureStrategy::default() } } } -/// 连接配置 +/// Connection configuration #[derive(Debug, Clone)] pub struct ConnectionConfig { - /// 连接超时时间（秒，默认：10） + /// Connection timeout in seconds (default: 10) pub connect_timeout: u64, - /// 请求超时时间（秒，默认：60） + /// Request timeout in seconds (default: 60) pub request_timeout: u64, - /// 最大解码消息大小（字节，默认：10MB） + /// Maximum decoding message size in bytes (default: 10MB) pub max_decoding_message_size: usize, } @@ -74,16 +53,14 @@ impl Default for ConnectionConfig { } } -/// 通用客户端配置 +/// Common client configuration #[derive(Debug, Clone)] pub struct StreamClientConfig { - /// 连接配置 + /// Connection configuration pub connection: ConnectionConfig, - /// 批处理配置 - pub batch: BatchConfig, - /// 背压配置 + /// Backpressure configuration pub backpressure: BackpressureConfig, - /// 是否启用性能监控（默认：false） + /// Whether performance monitoring is enabled (default: false) pub enable_metrics: bool, } @@ -91,7 +68,6 @@ impl Default for StreamClientConfig { fn default() -> Self { Self { connection: ConnectionConfig::default(), - batch: BatchConfig::default(), backpressure: BackpressureConfig::default(), enable_metrics: false, } @@ -99,31 +75,57 @@ impl Default for StreamClientConfig { } impl StreamClientConfig { - /// 创建高性能配置（适合高并发场景） - pub fn high_performance() -> Self { + /// Creates a high-throughput configuration optimized for high-concurrency scenarios. + /// + /// This configuration prioritizes throughput over latency by: + /// - Implementing a drop strategy for backpressure to avoid blocking + /// - Setting a large permit buffer (5,000) to handle burst traffic + /// + /// Ideal for scenarios where you need to process large volumes of data + /// and can tolerate occasional message drops during peak loads. + pub fn high_throughput() -> Self { Self { connection: ConnectionConfig::default(), - batch: BatchConfig { batch_size: 200, batch_timeout_ms: 5, enabled: true }, backpressure: BackpressureConfig { - channel_size: 20000, + permits: 5000, strategy: BackpressureStrategy::Drop, }, enable_metrics: false, } } - /// 创建低延迟配置（适合实时场景） + /// Creates a low-latency configuration optimized for real-time scenarios. + /// + /// This configuration prioritizes latency over throughput by: + /// - Processing events immediately without buffering + /// - Implementing a blocking backpressure strategy to ensure no data loss + /// - Setting minimal permits (1) to minimize memory usage + /// + /// Ideal for scenarios where every millisecond counts and you cannot + /// afford to lose any events, such as trading applications or real-time monitoring. pub fn low_latency() -> Self { Self { connection: ConnectionConfig::default(), - batch: BatchConfig { - batch_size: 10, - batch_timeout_ms: 1, - enabled: false, // 禁用批处理，即时处理 - }, + backpressure: BackpressureConfig { permits: 1, strategy: BackpressureStrategy::Block }, + enable_metrics: false, + } + } + + /// Creates an asynchronous processing configuration optimized for high-volume scenarios. + /// + /// This configuration balances throughput and reliability by: + /// - Implementing an async backpressure strategy for non-blocking operation + /// - Setting a balanced permit buffer (5,000) for steady flow + /// + /// Ideal for scenarios where you need sustained high throughput with + /// fire-and-forget semantics, such as data ingestion pipelines or + /// event streaming applications where some eventual consistency is acceptable. + pub fn async_processing() -> Self { + Self { + connection: ConnectionConfig::default(), backpressure: BackpressureConfig { - channel_size: 1000, - strategy: BackpressureStrategy::Block, + permits: 5000, + strategy: BackpressureStrategy::Async, }, enable_metrics: false, } diff --git a/src/streaming/common/constants.rs b/src/streaming/common/constants.rs index 0b253f6..e54309c 100644 --- a/src/streaming/common/constants.rs +++ b/src/streaming/common/constants.rs @@ -5,8 +5,6 @@ pub const DEFAULT_CONNECT_TIMEOUT: u64 = 10; pub const DEFAULT_REQUEST_TIMEOUT: u64 = 60; pub const DEFAULT_CHANNEL_SIZE: usize = 1000; pub const DEFAULT_MAX_DECODING_MESSAGE_SIZE: usize = 1024 * 1024 * 10; -pub const DEFAULT_BATCH_SIZE: usize = 100; -pub const DEFAULT_BATCH_TIMEOUT_MS: u64 = 5; // 性能监控相关常量 pub const DEFAULT_METRICS_WINDOW_SECONDS: u64 = 5; diff --git a/src/streaming/common/event_processor.rs b/src/streaming/common/event_processor.rs index e7ffe13..571a6b5 100644 --- a/src/streaming/common/event_processor.rs +++ b/src/streaming/common/event_processor.rs @@ -1,7 +1,9 @@ use std::sync::Arc; +use std::time::Instant; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Signature; +use tokio::sync::Semaphore; use crate::common::AnyResult; use crate::streaming::common::{ @@ -14,11 +16,11 @@ use crate::streaming::event_parser::EventParser; use crate::streaming::event_parser::{ core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, }; -use crate::streaming::grpc::{BackpressureConfig, BatchConfig, EventPretty}; +use crate::streaming::grpc::{BackpressureConfig, EventPretty}; use crate::streaming::shred::TransactionWithSlot; use once_cell::sync::OnceCell; -/// 事件处理器 +/// Event processor pub struct EventProcessor { pub(crate) metrics_manager: MetricsManager, pub(crate) config: ClientConfig, @@ -27,13 +29,15 @@ pub struct EventProcessor { pub(crate) event_type_filter: Option, pub(crate) callback: Option) + Send + Sync>>, pub(crate) backpressure_config: BackpressureConfig, - pub(crate) batch_config: BatchConfig, + /// Backpressure semaphore for controlling concurrent processing count + pub(crate) backpressure_semaphore: Arc, } impl EventProcessor { - /// 创建新的事件处理器 + /// Create a new event processor pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { let backpressure_config = config.backpressure.clone(); + let backpressure_semaphore = Arc::new(Semaphore::new(backpressure_config.permits)); Self { metrics_manager, config, @@ -41,8 +45,8 @@ impl EventProcessor { protocols: vec![], event_type_filter: None, backpressure_config, - batch_config: BatchConfig::default(), callback: None, + backpressure_semaphore, } } @@ -51,13 +55,15 @@ impl EventProcessor { protocols: Vec, event_type_filter: Option, backpressure_config: BackpressureConfig, - batch_config: BatchConfig, callback: Option) + Send + Sync>>, ) { self.protocols = protocols.clone(); self.event_type_filter = event_type_filter.clone(); + // Recreate semaphore if backpressure configuration changes + if self.backpressure_config.permits != backpressure_config.permits { + self.backpressure_semaphore = Arc::new(Semaphore::new(backpressure_config.permits)); + } self.backpressure_config = backpressure_config; - self.batch_config = batch_config; self.callback = callback; self.parser_cache .get_or_init(|| Arc::new(MutilEventParser::new(protocols, event_type_filter))); @@ -72,8 +78,73 @@ impl EventProcessor { event_pretty: EventPretty, bot_wallet: Option, ) -> AnyResult<()> { - self.process_grpc_event_transaction(event_pretty, bot_wallet).await?; - Ok(()) + // Backpressure control logic + let backpressure_start = Instant::now(); + let result = self.apply_backpressure_control(event_pretty, bot_wallet).await; + let backpressure_duration = backpressure_start.elapsed(); + + // Record backpressure-related metrics + self.metrics_manager.record_backpressure_metrics( + backpressure_duration, + result.is_ok(), + self.backpressure_semaphore.available_permits(), + ); + + result + } + + /// Apply backpressure control strategy + async fn apply_backpressure_control( + &self, + event_pretty: EventPretty, + bot_wallet: Option, + ) -> AnyResult<()> { + use crate::streaming::common::BackpressureStrategy; + + match self.backpressure_config.strategy { + BackpressureStrategy::Block => { + // Blocking strategy: acquire semaphore permit + let _permit = + self.backpressure_semaphore.acquire().await.map_err(|e| { + anyhow::anyhow!("Failed to acquire backpressure permit: {}", e) + })?; + self.process_grpc_event_transaction(event_pretty, bot_wallet).await + } + BackpressureStrategy::Drop => { + // Drop strategy: try to acquire permit, drop if failed + match self.backpressure_semaphore.try_acquire() { + Ok(_permit) => { + let result = + self.process_grpc_event_transaction(event_pretty, bot_wallet).await; + result + } + Err(_) => { + // Record dropped event + self.metrics_manager.increment_dropped_events(); + Ok(()) + } + } + } + BackpressureStrategy::Async => { + // Async strategy: process asynchronously regardless of permits + self.spawn_async_processing(event_pretty, bot_wallet).await; + Ok(()) + } + } + } + + /// Process event asynchronously (without waiting for semaphore permit) + async fn spawn_async_processing(&self, event_pretty: EventPretty, bot_wallet: Option) { + let processor = self.clone(); + + tokio::spawn(async move { + // Async strategy: no semaphore control, allow unlimited concurrency + // Execute actual event processing directly + if let Err(e) = processor.process_grpc_event_transaction(event_pretty, bot_wallet).await + { + log::error!("Error in async event processing: {}", e); + } + }); } async fn process_grpc_event_transaction( @@ -81,6 +152,9 @@ impl EventProcessor { event_pretty: EventPretty, bot_wallet: Option, ) -> AnyResult<()> { + if self.callback.is_none() { + return Ok(()); + } match event_pretty { EventPretty::Account(account_pretty) => { self.metrics_manager.add_account_process_count(); @@ -105,40 +179,36 @@ impl EventProcessor { self.metrics_manager.add_tx_process_count(); let slot = transaction_pretty.slot; let signature = transaction_pretty.signature; - // 使用缓存获取解析器 + let tx = transaction_pretty.tx; + let block_time = transaction_pretty.block_time; + let program_received_time_us = transaction_pretty.program_received_time_us; + let transaction_index = transaction_pretty.transaction_index; + // Use cache to get parser let parser = self.get_parser(); - let all_events = parser - .parse_transaction( - transaction_pretty.tx.clone(), + let callback = self.callback.clone().unwrap(); + let metrics_manager = self.metrics_manager.clone(); + let adapter_callback = Arc::new(move |event: Box| { + let processing_time_us = event.program_handle_time_consuming_us() as f64; + callback(event); + metrics_manager.update_metrics( + MetricsEventType::Transaction, + 1, + processing_time_us, + Some(signature), + ); + }); + parser + .parse_transaction_owned( + tx, signature, Some(slot), - transaction_pretty.block_time, - transaction_pretty.program_received_time_us, + block_time, + program_received_time_us, bot_wallet, - transaction_pretty.transaction_index, + transaction_index, + adapter_callback, ) - .await - .unwrap_or_else(|_e| vec![]); - - let mut all_time_consuming_us = 0; - let event_count = all_events.len(); - - // 为所有事件设置交易索引 - for mut event in all_events { - event.set_program_handle_time_consuming_us( - chrono::Utc::now().timestamp_micros() - event.program_received_time_us(), - ); - all_time_consuming_us += event.program_handle_time_consuming_us(); - self.invoke_callback(event); - } - - // 更新性能指标 - self.update_metrics( - MetricsEventType::Transaction, - event_count as u64, - all_time_consuming_us as f64, - Some(signature), - ); + .await?; } EventPretty::BlockMeta(block_meta_pretty) => { self.metrics_manager.add_block_meta_process_count(); @@ -167,7 +237,7 @@ impl EventProcessor { } } - /// 即时处理单个交易 + /// Process a single transaction immediately pub async fn process_shred_transaction_immediate( &self, transaction_with_slot: TransactionWithSlot, @@ -176,55 +246,129 @@ impl EventProcessor { self.process_shred_transaction(transaction_with_slot, bot_wallet).await } + /// Process shred transaction with backpressure control and performance monitoring + pub async fn process_shred_transaction_with_metrics( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { + // Backpressure control logic + let backpressure_start = Instant::now(); + let result = self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await; + let backpressure_duration = backpressure_start.elapsed(); + + // Record backpressure-related metrics + self.metrics_manager.record_backpressure_metrics( + backpressure_duration, + result.is_ok(), + self.backpressure_semaphore.available_permits(), + ); + + result + } + + /// Apply shred backpressure control strategy + async fn apply_shred_backpressure_control( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) -> AnyResult<()> { + use crate::streaming::common::BackpressureStrategy; + + match self.backpressure_config.strategy { + BackpressureStrategy::Block => { + // Blocking strategy: acquire semaphore permit + let _permit = + self.backpressure_semaphore.acquire().await.map_err(|e| { + anyhow::anyhow!("Failed to acquire backpressure permit: {}", e) + })?; + self.process_shred_transaction(transaction_with_slot, bot_wallet).await + } + BackpressureStrategy::Drop => { + // Drop strategy: try to acquire permit, drop if failed + match self.backpressure_semaphore.try_acquire() { + Ok(_permit) => { + let result = + self.process_shred_transaction(transaction_with_slot, bot_wallet).await; + result + } + Err(_) => { + // Record dropped event + self.metrics_manager.increment_dropped_events(); + Ok(()) + } + } + } + BackpressureStrategy::Async => { + // Async strategy: process asynchronously regardless of permits + self.spawn_async_shred_processing(transaction_with_slot, bot_wallet).await; + Ok(()) + } + } + } + + /// Process shred event asynchronously (without waiting for semaphore permit) + async fn spawn_async_shred_processing( + &self, + transaction_with_slot: TransactionWithSlot, + bot_wallet: Option, + ) { + let processor = self.clone(); + + tokio::spawn(async move { + // Async strategy: no semaphore control, allow unlimited concurrency + // Execute actual event processing directly + if let Err(e) = + processor.process_shred_transaction(transaction_with_slot, bot_wallet).await + { + log::error!("Error in async shred event processing: {}", e); + } + }); + } + pub async fn process_shred_transaction( &self, transaction_with_slot: TransactionWithSlot, bot_wallet: Option, ) -> AnyResult<()> { + if self.callback.is_none() { + return Ok(()); + } self.metrics_manager.add_tx_process_count(); - let program_received_time_us = chrono::Utc::now().timestamp_micros(); + let tx = transaction_with_slot.transaction; + let slot = transaction_with_slot.slot; - let versioned_tx = transaction_with_slot.transaction; - let signature = versioned_tx.signatures[0]; - - // 获取缓存的解析器 + let signature = tx.signatures[0]; + let program_received_time_us = transaction_with_slot.program_received_time_us; + // Use cache to get parser let parser = self.get_parser(); + let callback = self.callback.clone().unwrap(); + let metrics_manager = self.metrics_manager.clone(); - let all_events = parser - .parse_versioned_transaction( - &versioned_tx, + let adapter_callback = Arc::new(move |event: Box| { + let processing_time_us = event.program_handle_time_consuming_us() as f64; + callback(event); + metrics_manager.update_metrics( + MetricsEventType::Transaction, + 1, + processing_time_us, + Some(signature), + ); + }); + + parser + .parse_versioned_transaction_owned( + tx, signature, Some(slot), None, program_received_time_us, bot_wallet, None, + &[], + adapter_callback, ) - .await - .unwrap_or_else(|_e| vec![]); - - let mut max_time_consuming_us = 0; - - // 保存事件数量用于日志记录 - let event_count = all_events.len(); - - // 即时处理事件 - for mut event in all_events { - event.set_program_handle_time_consuming_us( - chrono::Utc::now().timestamp_micros() - event.program_received_time_us(), - ); - max_time_consuming_us = - max_time_consuming_us.max(event.program_handle_time_consuming_us()); - self.invoke_callback(event); - } - - // 实际调用性能指标更新 - self.update_metrics( - MetricsEventType::Transaction, - event_count as u64, - max_time_consuming_us as f64, - Some(signature), - ); + .await?; Ok(()) } @@ -240,7 +384,7 @@ impl EventProcessor { } } -// 实现 Clone trait 以支持模块间共享 +// Implement Clone trait to support sharing between modules impl Clone for EventProcessor { fn clone(&self) -> Self { Self { @@ -250,8 +394,8 @@ impl Clone for EventProcessor { protocols: self.protocols.clone(), event_type_filter: self.event_type_filter.clone(), backpressure_config: self.backpressure_config.clone(), - batch_config: self.batch_config.clone(), callback: self.callback.clone(), + backpressure_semaphore: self.backpressure_semaphore.clone(), } } } diff --git a/src/streaming/common/metrics.rs b/src/streaming/common/metrics.rs index a5df787..dd02697 100644 --- a/src/streaming/common/metrics.rs +++ b/src/streaming/common/metrics.rs @@ -108,15 +108,21 @@ impl AtomicEventMetrics { struct AtomicProcessingTimeStats { min_time_bits: AtomicU64, max_time_bits: AtomicU64, - total_time_us: AtomicU64, // 存储微秒的整数部分 + max_time_timestamp_nanos: AtomicU64, // 最大值更新时间戳（纳秒） + total_time_us: AtomicU64, // 存储微秒的整数部分 total_events: AtomicU64, } impl AtomicProcessingTimeStats { fn new() -> Self { + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; + Self { min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()), max_time_bits: AtomicU64::new(0), + max_time_timestamp_nanos: AtomicU64::new(now_nanos), total_time_us: AtomicU64::new(0), total_events: AtomicU64::new(0), } @@ -126,6 +132,9 @@ impl AtomicProcessingTimeStats { #[inline] fn update(&self, time_us: f64, event_count: u64) { let time_bits = time_us.to_bits(); + let now_nanos = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + as u64; // 更新最小值（使用 compare_exchange_weak 循环） let mut current_min = self.min_time_bits.load(Ordering::Relaxed); @@ -141,8 +150,20 @@ impl AtomicProcessingTimeStats { } } - // 更新最大值 + // 更新最大值，检查时间差并在超过10秒时清零 let mut current_max = self.max_time_bits.load(Ordering::Relaxed); + let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed); + + // 检查最大值的时间戳是否超过10秒（10_000_000_000纳秒） + let time_diff_nanos = now_nanos.saturating_sub(max_timestamp); + if time_diff_nanos > 10_000_000_000 { + // 超过10秒，清零最大值 + self.max_time_bits.store(0, Ordering::Relaxed); + self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + current_max = 0; + } + + // 如果当前时间大于最大值，更新最大值和时间戳 while time_bits > current_max { match self.max_time_bits.compare_exchange_weak( current_max, @@ -150,7 +171,11 @@ impl AtomicProcessingTimeStats { Ordering::Relaxed, Ordering::Relaxed, ) { - Ok(_) => break, + Ok(_) => { + // 成功更新最大值，同时更新时间戳 + self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + break; + } Err(x) => current_max = x, } } @@ -198,6 +223,18 @@ pub struct EventMetricsSnapshot { pub events_per_second: f64, } +/// 背压指标快照 +#[derive(Debug, Clone)] +pub struct BackpressureMetricsSnapshot { + pub total_duration_us: u64, + pub success_count: u64, + pub failure_count: u64, + pub min_permits: u64, + pub max_permits: u64, + pub avg_duration_us: f64, + pub success_rate: f64, +} + /// 兼容性结构 - 完整的性能指标 #[derive(Debug, Clone)] pub struct PerformanceMetrics { @@ -206,6 +243,8 @@ pub struct PerformanceMetrics { pub account_metrics: EventMetricsSnapshot, pub block_meta_metrics: EventMetricsSnapshot, pub processing_stats: ProcessingTimeStats, + pub backpressure_metrics: BackpressureMetricsSnapshot, + pub dropped_events_count: u64, } impl PerformanceMetrics { @@ -214,6 +253,15 @@ impl PerformanceMetrics { let default_metrics = EventMetricsSnapshot { process_count: 0, events_processed: 0, events_per_second: 0.0 }; let default_stats = ProcessingTimeStats { min_us: 0.0, max_us: 0.0, avg_us: 0.0 }; + let default_backpressure = BackpressureMetricsSnapshot { + total_duration_us: 0, + success_count: 0, + failure_count: 0, + min_permits: 0, + max_permits: 0, + avg_duration_us: 0.0, + success_rate: 0.0, + }; Self { uptime: std::time::Duration::ZERO, @@ -221,6 +269,8 @@ impl PerformanceMetrics { account_metrics: default_metrics.clone(), block_meta_metrics: default_metrics, processing_stats: default_stats, + backpressure_metrics: default_backpressure, + dropped_events_count: 0, } } } @@ -231,6 +281,14 @@ pub struct HighPerformanceMetrics { start_nanos: u64, event_metrics: [AtomicEventMetrics; 3], processing_stats: AtomicProcessingTimeStats, + // 背压相关指标 + backpressure_total_duration_us: AtomicU64, + backpressure_success_count: AtomicU64, + backpressure_failure_count: AtomicU64, + backpressure_min_permits: AtomicU64, + backpressure_max_permits: AtomicU64, + // 丢弃事件指标 + dropped_events_count: AtomicU64, } impl HighPerformanceMetrics { @@ -247,6 +305,14 @@ impl HighPerformanceMetrics { AtomicEventMetrics::new(now_nanos), ], processing_stats: AtomicProcessingTimeStats::new(), + // 初始化背压相关指标 + backpressure_total_duration_us: AtomicU64::new(0), + backpressure_success_count: AtomicU64::new(0), + backpressure_failure_count: AtomicU64::new(0), + backpressure_min_permits: AtomicU64::new(u64::MAX), // 初始化为最大值，便于后续比较 + backpressure_max_permits: AtomicU64::new(0), + // 初始化丢弃事件指标 + dropped_events_count: AtomicU64::new(0), } } @@ -275,6 +341,38 @@ impl HighPerformanceMetrics { self.processing_stats.get_stats() } + /// 获取背压指标快照 + #[inline] + pub fn get_backpressure_metrics(&self) -> BackpressureMetricsSnapshot { + let total_duration_us = self.backpressure_total_duration_us.load(Ordering::Relaxed); + let success_count = self.backpressure_success_count.load(Ordering::Relaxed); + let failure_count = self.backpressure_failure_count.load(Ordering::Relaxed); + let min_permits = self.backpressure_min_permits.load(Ordering::Relaxed); + let max_permits = self.backpressure_max_permits.load(Ordering::Relaxed); + + let total_count = success_count + failure_count; + let avg_duration_us = + if total_count > 0 { total_duration_us as f64 / total_count as f64 } else { 0.0 }; + let success_rate = + if total_count > 0 { success_count as f64 / total_count as f64 } else { 0.0 }; + + BackpressureMetricsSnapshot { + total_duration_us, + success_count, + failure_count, + min_permits: if min_permits == u64::MAX { 0 } else { min_permits }, + max_permits, + avg_duration_us, + success_rate, + } + } + + /// 获取丢弃事件计数 + #[inline] + pub fn get_dropped_events_count(&self) -> u64 { + self.dropped_events_count.load(Ordering::Relaxed) + } + /// 计算实时每秒事件数（非阻塞） fn calculate_real_time_eps(&self, event_type: EventType) -> f64 { let now_nanos = @@ -443,11 +541,43 @@ impl MetricsManager { self.metrics.get_processing_stats() } + /// 获取背压指标 + pub fn get_backpressure_metrics(&self) -> BackpressureMetricsSnapshot { + self.metrics.get_backpressure_metrics() + } + + /// 获取丢弃事件计数 + pub fn get_dropped_events_count(&self) -> u64 { + self.metrics.get_dropped_events_count() + } + /// 打印性能指标（非阻塞） pub fn print_metrics(&self) { println!("\n📊 {} Performance Metrics", self.stream_name); println!(" Run Time: {:?}", self.get_uptime()); + // 打印背压指标表格 + let backpressure = self.get_backpressure_metrics(); + if backpressure.success_count > 0 || backpressure.failure_count > 0 { + println!("\n🚦 Backpressure Metrics"); + println!("┌──────────────────────┬─────────────┐"); + println!("│ Metric │ Value │"); + println!("├──────────────────────┼─────────────┤"); + println!("│ Success Count │ {:11} │", backpressure.success_count); + println!("│ Failure Count │ {:11} │", backpressure.failure_count); + println!("│ Success Rate │ {:11.2} │", backpressure.success_rate * 100.0); + println!("│ Avg Duration (ms) │ {:11.2} │", backpressure.avg_duration_us / 1000.0); + println!("│ Min Permits │ {:11} │", backpressure.min_permits); + println!("│ Max Permits │ {:11} │", backpressure.max_permits); + println!("└──────────────────────┴─────────────┘"); + } + + // 打印丢弃事件指标 + let dropped_count = self.get_dropped_events_count(); + if dropped_count > 0 { + println!("\n⚠️ Dropped Events: {}", dropped_count); + } + // 打印事件指标表格 println!("┌─────────────┬──────────────┬──────────────────┬─────────────────┐"); println!("│ Event Type │ Process Count│ Events Processed │ Events/Second │"); @@ -469,13 +599,14 @@ impl MetricsManager { // 打印处理时间统计表格 let stats = self.get_processing_stats(); println!("\n⏱️ Processing Time Statistics"); - println!("┌─────────────────────┬─────────────┐"); - println!("│ Metric │ Value (us) │"); - println!("├─────────────────────┼─────────────┤"); - println!("│ Average │ {:9.2} │", stats.avg_us); - println!("│ Minimum │ {:9.2} │", stats.min_us); - println!("│ Maximum │ {:9.2} │", stats.max_us); - println!("└─────────────────────┴─────────────┘"); + println!("┌───────────────────────┬─────────────┐"); + println!("│ Metric │ Value (us) │"); + println!("├───────────────────────┼─────────────┤"); + println!("│ Average │ {:9.2} │", stats.avg_us); + println!("│ Minimum │ {:9.2} │", stats.min_us); + println!("│ Maximum within 10s │ {:9.2} │", stats.max_us); + println!("└───────────────────────┴─────────────┘"); + println!(); } @@ -517,6 +648,8 @@ impl MetricsManager { account_metrics: self.get_event_metrics(EventType::Account), block_meta_metrics: self.get_event_metrics(EventType::BlockMeta), processing_stats: self.get_processing_stats(), + backpressure_metrics: self.metrics.get_backpressure_metrics(), + dropped_events_count: self.metrics.get_dropped_events_count(), } } @@ -550,6 +683,110 @@ impl MetricsManager { self.record_events(event_type, events_processed, processing_time_us); self.log_slow_processing(processing_time_us, events_processed as usize, signature); } + + /// 记录背压相关的metrics + #[inline] + pub fn record_backpressure_metrics( + &self, + backpressure_duration: std::time::Duration, + success: bool, + available_permits: usize, + ) { + if !self.enable_metrics { + return; + } + + let duration_us = backpressure_duration.as_micros() as u64; + let permits = available_permits as u64; + + // 记录总持续时间 + self.metrics.backpressure_total_duration_us.fetch_add(duration_us, Ordering::Relaxed); + + // 记录成功/失败计数 + if success { + self.metrics.backpressure_success_count.fetch_add(1, Ordering::Relaxed); + } else { + self.metrics.backpressure_failure_count.fetch_add(1, Ordering::Relaxed); + } + + // 更新最小许可数（使用 compare_exchange_weak 循环） + let mut current_min = self.metrics.backpressure_min_permits.load(Ordering::Relaxed); + while permits < current_min { + match self.metrics.backpressure_min_permits.compare_exchange_weak( + current_min, + permits, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(x) => current_min = x, + } + } + + // 更新最大许可数 + let mut current_max = self.metrics.backpressure_max_permits.load(Ordering::Relaxed); + while permits > current_max { + match self.metrics.backpressure_max_permits.compare_exchange_weak( + current_max, + permits, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(x) => current_max = x, + } + } + + // 记录慢背压操作的日志 + if duration_us > 10_000 { + // 超过10ms的背压认为是慢操作 + log::warn!( + "{} slow backpressure: {:.2}ms, success: {}, available_permits: {}", + self.stream_name, + duration_us as f64 / 1000.0, + success, + available_permits + ); + } + } + + /// 增加丢弃事件计数 + #[inline] + pub fn increment_dropped_events(&self) { + if !self.enable_metrics { + return; + } + + // 原子地增加丢弃事件计数 + let new_count = self.metrics.dropped_events_count.fetch_add(1, Ordering::Relaxed) + 1; + + // 每丢弃1000个事件记录一次警告日志 + if new_count % 1000 == 0 { + log::warn!("{} dropped events count reached: {}", self.stream_name, new_count); + } + } + + /// 批量增加丢弃事件计数 + #[inline] + pub fn increment_dropped_events_by(&self, count: u64) { + if !self.enable_metrics || count == 0 { + return; + } + + // 原子地增加丢弃事件计数 + let new_count = self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count; + + // 记录批量丢弃事件的日志 + if count > 1 { + log::warn!("{} dropped batch of {} events, total dropped: {}", + self.stream_name, count, new_count); + } + + // 每丢弃1000个事件记录一次警告日志 + if new_count % 1000 == 0 || (new_count / 1000) != ((new_count - count) / 1000) { + log::warn!("{} dropped events count reached: {}", self.stream_name, new_count); + } + } } impl Clone for MetricsManager { diff --git a/src/streaming/event_parser/core/global_state.rs b/src/streaming/event_parser/core/global_state.rs new file mode 100644 index 0000000..5e75c0f --- /dev/null +++ b/src/streaming/event_parser/core/global_state.rs @@ -0,0 +1,141 @@ +use solana_sdk::pubkey::Pubkey; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Global state management, thread-safe implementation without locks +pub struct GlobalState { + /// Last processed slot + last_slot: AtomicU64, + /// Developer address array + dev_addresses: parking_lot::RwLock>, + /// Bonk developer address array + bonk_dev_addresses: parking_lot::RwLock>, +} + +impl GlobalState { + /// Create a new global state instance + pub fn new() -> Self { + Self { + last_slot: AtomicU64::new(0), + dev_addresses: parking_lot::RwLock::new(Vec::new()), + bonk_dev_addresses: parking_lot::RwLock::new(Vec::new()), + } + } + + /// Get current slot + pub fn get_last_slot(&self) -> u64 { + self.last_slot.load(Ordering::Relaxed) + } + + /// Update slot, clear arrays if slot changes + pub fn update_slot(&self, new_slot: u64) { + let old_slot = self.last_slot.swap(new_slot, Ordering::Relaxed); + + if old_slot != new_slot { + // Clear arrays when slot changes + let mut dev_addresses = self.dev_addresses.write(); + let mut bonk_dev_addresses = self.bonk_dev_addresses.write(); + + dev_addresses.clear(); + bonk_dev_addresses.clear(); + } + } + + /// Add developer address + pub fn add_dev_address(&self, address: Pubkey) { + let mut dev_addresses = self.dev_addresses.write(); + if !dev_addresses.contains(&address) { + dev_addresses.push(address); + } + } + + /// Check if address is a developer address + pub fn is_dev_address(&self, address: &Pubkey) -> bool { + let dev_addresses = self.dev_addresses.read(); + dev_addresses.contains(address) + } + + /// Add Bonk developer address + pub fn add_bonk_dev_address(&self, address: Pubkey) { + let mut bonk_dev_addresses = self.bonk_dev_addresses.write(); + if !bonk_dev_addresses.contains(&address) { + bonk_dev_addresses.push(address); + } + } + + /// Check if address is a Bonk developer address + pub fn is_bonk_dev_address(&self, address: &Pubkey) -> bool { + let bonk_dev_addresses = self.bonk_dev_addresses.read(); + bonk_dev_addresses.contains(address) + } + + /// Get all developer addresses + pub fn get_dev_addresses(&self) -> Vec { + let dev_addresses = self.dev_addresses.read(); + dev_addresses.clone() + } + + /// Get all Bonk developer addresses + pub fn get_bonk_dev_addresses(&self) -> Vec { + let bonk_dev_addresses = self.bonk_dev_addresses.read(); + bonk_dev_addresses.clone() + } + + /// Clear all data + pub fn clear_all_data(&self) { + let mut dev_addresses = self.dev_addresses.write(); + let mut bonk_dev_addresses = self.bonk_dev_addresses.write(); + + dev_addresses.clear(); + bonk_dev_addresses.clear(); + } +} + +impl Default for GlobalState { + fn default() -> Self { + Self::new() + } +} + +/// Global state instance +static GLOBAL_STATE: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(GlobalState::new); + +/// Get global state instance +pub fn get_global_state() -> &'static GlobalState { + &GLOBAL_STATE +} + +/// Convenience function: Update slot +pub fn update_slot(slot: u64) { + get_global_state().update_slot(slot); +} + +/// Convenience function: Add developer address +pub fn add_dev_address(address: Pubkey) { + get_global_state().add_dev_address(address); +} + +/// Convenience function: Check if address is a developer address +pub fn is_dev_address(address: &Pubkey) -> bool { + get_global_state().is_dev_address(address) +} + +/// Convenience function: Add Bonk developer address +pub fn add_bonk_dev_address(address: Pubkey) { + get_global_state().add_bonk_dev_address(address); +} + +/// Convenience function: Check if address is a Bonk developer address +pub fn is_bonk_dev_address(address: &Pubkey) -> bool { + get_global_state().is_bonk_dev_address(address) +} + +/// Convenience function: Get all developer addresses +pub fn get_dev_addresses() -> Vec { + get_global_state().get_dev_addresses() +} + +/// Convenience function: Get all Bonk developer addresses +pub fn get_bonk_dev_addresses() -> Vec { + get_global_state().get_bonk_dev_addresses() +} diff --git a/src/streaming/event_parser/core/macros.rs b/src/streaming/event_parser/core/macros.rs index 9ef239e..724ccc0 100644 --- a/src/streaming/event_parser/core/macros.rs +++ b/src/streaming/event_parser/core/macros.rs @@ -15,15 +15,6 @@ macro_rules! impl_event_parser_delegate { ($parser_type:ty) => { #[async_trait::async_trait] impl $crate::streaming::event_parser::core::traits::EventParser for $parser_type { - fn inner_instruction_configs( - &self, - ) -> std::collections::HashMap< - Vec, - Vec<$crate::streaming::event_parser::core::traits::GenericEventParseConfig>, - > { - self.inner.inner_instruction_configs() - } - fn instruction_configs( &self, ) -> std::collections::HashMap< @@ -42,8 +33,8 @@ macro_rules! impl_event_parser_delegate { program_received_time_us: i64, outer_index: i64, inner_index: Option, - bot_wallet: Option, transaction_index: Option, + config: &GenericEventParseConfig, ) -> Vec> { self.inner.parse_events_from_inner_instruction( inner_instruction, @@ -53,8 +44,8 @@ macro_rules! impl_event_parser_delegate { program_received_time_us, outer_index, inner_index, - bot_wallet, transaction_index, + config, ) } @@ -70,7 +61,13 @@ macro_rules! impl_event_parser_delegate { inner_index: Option, bot_wallet: Option, transaction_index: Option, - ) -> Vec> { + inner_instructions: Option<&solana_transaction_status::InnerInstructions>, + callback: std::sync::Arc< + dyn for<'a> Fn(&'a Box) + + Send + + Sync, + >, + ) -> anyhow::Result<()> { self.inner.parse_events_from_instruction( instruction, accounts, @@ -82,6 +79,8 @@ macro_rules! impl_event_parser_delegate { inner_index, bot_wallet, transaction_index, + inner_instructions, + callback, ) } diff --git a/src/streaming/event_parser/core/mod.rs b/src/streaming/event_parser/core/mod.rs index cbe0602..2e9f50e 100755 --- a/src/streaming/event_parser/core/mod.rs +++ b/src/streaming/event_parser/core/mod.rs @@ -2,4 +2,5 @@ pub mod common_event_parser; pub mod traits; pub mod account_event_parser; pub mod macros; +pub mod global_state; pub use traits::{EventParser, UnifiedEvent}; diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index b37da1a..db74152 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -1,15 +1,23 @@ -use anyhow::Result; use prost_types::Timestamp; +use solana_sdk::bs58; use solana_sdk::signature::Signature; use solana_sdk::{ instruction::CompiledInstruction, pubkey::Pubkey, transaction::VersionedTransaction, }; -use solana_transaction_status::{InnerInstructions, TransactionWithStatusMeta}; +use solana_transaction_status::{ + EncodedConfirmedTransactionWithStatusMeta, InnerInstruction, InnerInstructions, + TransactionWithStatusMeta, UiInstruction, +}; use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; +use super::global_state::{add_dev_address, is_dev_address, update_slot}; + use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData}; +use crate::streaming::event_parser::core::global_state::{ + add_bonk_dev_address, is_bonk_dev_address, +}; use crate::streaming::event_parser::protocols::pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}; use crate::streaming::event_parser::{ common::{utils::*, EventMetadata, EventType, ProtocolType}, @@ -73,8 +81,6 @@ pub trait UnifiedEvent: Debug + Send + Sync { /// 事件解析器trait - 定义了事件解析的核心方法 #[async_trait::async_trait] pub trait EventParser: Send + Sync { - /// 获取内联指令解析配置 - fn inner_instruction_configs(&self) -> HashMap, Vec>; /// 获取指令解析配置 fn instruction_configs(&self) -> HashMap, Vec>; /// 从内联指令中解析事件数据 @@ -88,8 +94,8 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, outer_index: i64, inner_index: Option, - bot_wallet: Option, transaction_index: Option, + config: &GenericEventParseConfig, ) -> Vec>; /// 从指令中解析事件数据 @@ -106,7 +112,9 @@ pub trait EventParser: Send + Sync { inner_index: Option, bot_wallet: Option, transaction_index: Option, - ) -> Vec>; + inner_instructions: Option<&InnerInstructions>, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()>; /// 从VersionedTransaction中解析指令事件的通用方法 #[allow(clippy::too_many_arguments)] @@ -121,13 +129,11 @@ pub trait EventParser: Send + Sync { inner_instructions: &[InnerInstructions], bot_wallet: Option, transaction_index: Option, - ) -> Result>> { - // 预分配容量，避免动态扩容 - let mut instruction_events = Vec::with_capacity(16); + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { // 获取交易的指令和账户 let compiled_instructions = transaction.message.instructions(); let mut accounts: Vec = accounts.to_vec(); - // 检查交易中是否包含程序 let has_program = accounts.iter().any(|account| self.should_handle(account)); if has_program { @@ -142,47 +148,60 @@ pub trait EventParser: Send + Sync { accounts.push(Pubkey::default()); } } - if let Ok(mut events) = self - .parse_instruction( - instruction, - &accounts, - signature, - slot, - block_time, - program_received_time_us, - index as i64, - None, - bot_wallet, - Some(index as u64), - ) - .await - { - if !events.is_empty() { - if let Some(inn) = - inner_instructions.iter().find(|inner_instruction| { - inner_instruction.index == index as u8 - }) - { - events.iter_mut().for_each(|event| { - let swap_data = parse_swap_data_from_next_instructions( - event.as_ref(), - inn, - -1_i8, - &accounts, - ); - if let Some(swap_data) = swap_data { - event.set_swap_data(swap_data); - } - }); - } - instruction_events.extend(events); - } - } + let inner_instructions = inner_instructions + .iter() + .find(|inner_instruction| inner_instruction.index == index as u8); + self.parse_instruction( + instruction, + &accounts, + signature, + slot, + block_time, + program_received_time_us, + index as i64, + None, + bot_wallet, + transaction_index, + inner_instructions, + Arc::clone(&callback), + ) + .await?; } } } } - Ok(instruction_events) + Ok(()) + } + + async fn parse_versioned_transaction_owned( + &self, + versioned_tx: VersionedTransaction, + signature: Signature, + slot: Option, + block_time: Option, + program_received_time_us: i64, + bot_wallet: Option, + transaction_index: Option, + inner_instructions: &[InnerInstructions], + callback: Arc) + Send + Sync>, + ) -> anyhow::Result<()> { + // 创建适配器回调，将所有权回调转换为引用回调 + let adapter_callback = Arc::new(move |event: &Box| { + callback(event.clone_boxed()); + }); + self.parse_versioned_transaction( + &versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + bot_wallet, + transaction_index, + inner_instructions, + adapter_callback, + ) + .await?; + Ok(()) } async fn parse_versioned_transaction( @@ -194,23 +213,54 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, bot_wallet: Option, transaction_index: Option, - ) -> Result>> { + inner_instructions: &[InnerInstructions], + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let accounts: Vec = versioned_tx.message.static_account_keys().to_vec(); - let events = self - .parse_instruction_events_from_versioned_transaction( - versioned_tx, - signature, - slot, - block_time, - program_received_time_us, - &accounts, - &[], - bot_wallet, - transaction_index, - ) - .await - .unwrap_or_else(|_e| vec![]); - Ok(self.process_events(events, bot_wallet)) + self.parse_instruction_events_from_versioned_transaction( + versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + &accounts, + inner_instructions, + bot_wallet, + transaction_index, + callback, + ) + .await?; + Ok(()) + } + + /// 解析交易，使用所有权语义的回调以避免不必要的克隆 + async fn parse_transaction_owned( + &self, + tx: TransactionWithStatusMeta, + signature: Signature, + slot: Option, + block_time: Option, + program_received_time_us: i64, + bot_wallet: Option, + transaction_index: Option, + callback: Arc) + Send + Sync>, + ) -> anyhow::Result<()> { + // 创建适配器回调，将所有权回调转换为引用回调 + let adapter_callback = Arc::new(move |event: &Box| { + callback(event.clone_boxed()); + }); + // 调用原始方法 + self.parse_transaction( + tx, + signature, + slot, + block_time, + program_received_time_us, + bot_wallet, + transaction_index, + adapter_callback, + ) + .await } async fn parse_transaction( @@ -222,10 +272,10 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, bot_wallet: Option, transaction_index: Option, - ) -> Result>> { + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let versioned_tx = tx.get_transaction(); let meta = tx.get_status_meta(); - let mut address_table_lookups: Vec = vec![]; let mut inner_instructions: Vec = vec![]; if let Some(meta) = meta { @@ -237,269 +287,183 @@ pub trait EventParser: Send + Sync { meta.loaded_addresses.writable.into_iter().chain(meta.loaded_addresses.readonly), ); } - let mut accounts = Vec::with_capacity( versioned_tx.message.static_account_keys().len() + address_table_lookups.len(), ); accounts.extend_from_slice(versioned_tx.message.static_account_keys()); accounts.extend(address_table_lookups); - // 使用 Arc 包装共享数据，避免不必要的克隆 let accounts_arc = Arc::new(accounts); let inner_instructions_arc = Arc::new(inner_instructions); + // 解析指令事件 + self.parse_instruction_events_from_versioned_transaction( + &versioned_tx, + signature, + slot, + block_time, + program_received_time_us, + &accounts_arc, + &inner_instructions_arc, + bot_wallet, + transaction_index, + callback.clone(), + ) + .await?; - // 预分配容量，避免动态扩容 - let mut instruction_events: Vec> = Vec::with_capacity(16); - let mut inner_instruction_events: Vec> = Vec::with_capacity(8); - - let accounts_for_task1 = Arc::clone(&accounts_arc); - let inner_instructions_for_task1 = Arc::clone(&inner_instructions_arc); - let task1 = async move { - self.parse_instruction_events_from_versioned_transaction( - &versioned_tx, - signature, - slot, - block_time, - program_received_time_us, - &accounts_for_task1, - &inner_instructions_for_task1, - bot_wallet, - transaction_index, - ) - .await - .unwrap_or_else(|_e| vec![]) - }; - - // 解析内联指令事件 - let inner_instructions_for_task2 = Arc::clone(&inner_instructions_arc); - let accounts_for_task2_1 = Arc::clone(&accounts_arc); - let accounts_for_task2_2 = Arc::clone(&accounts_arc); - - let mut task2_params = Vec::with_capacity(inner_instructions_for_task2.len() * 5); - for inner_instruction in inner_instructions_for_task2.iter() { + // 解析嵌套指令事件 + for inner_instruction in inner_instructions_arc.iter() { for (index, instruction) in inner_instruction.instructions.iter().enumerate() { - task2_params.push(( + self.parse_instruction( &instruction.instruction, + &accounts_arc, signature, slot, block_time, program_received_time_us, inner_instruction.index as i64, Some(index as i64), - inner_instruction, - )); + bot_wallet, + transaction_index, + Some(&inner_instruction), + callback.clone(), + ) + .await?; } } - // 转换为 Arc<[T]> 更轻量 - let task2_params: Arc<[_]> = task2_params.into(); - let task2_params_clone = Arc::clone(&task2_params); - let task2_1 = async move { - let mut instruction_events: Vec> = Vec::with_capacity(16); - for ( - instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - inner_instruction, - ) in task2_params_clone.iter() - { - if let Ok(mut events) = self - .parse_instruction( - instruction, - &accounts_for_task2_1, - *signature, - *slot, - *block_time, - *program_received_time_us, - *outer_index, - *inner_index, - bot_wallet, - transaction_index, - ) - .await - { - if !events.is_empty() { - events.iter_mut().for_each(|event| { - let swap_data = parse_swap_data_from_next_instructions( - event.as_ref(), - &inner_instruction, - inner_index.unwrap_or_default() as i8, - &accounts_for_task2_1, - ); - if let Some(swap_data) = swap_data { - event.set_swap_data(swap_data); - } - }); - instruction_events.extend(events); - } - } - } - instruction_events - }; - let task2_2 = async move { - let mut inner_instruction_events: Vec> = Vec::with_capacity(8); - for ( - instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - inner_instruction, - ) in task2_params.iter() - { - if let Ok(mut events) = self - .parse_inner_instruction( - instruction, - *signature, - *slot, - *block_time, - *program_received_time_us, - *outer_index, - *inner_index, - bot_wallet, - transaction_index, - ) - .await - { - if !events.is_empty() { - events.iter_mut().for_each(|event| { - let swap_data = parse_swap_data_from_next_instructions( - event.as_ref(), - &inner_instruction, - inner_index.unwrap_or_default() as i8, - &accounts_for_task2_2, - ); - if let Some(swap_data) = swap_data { - event.set_swap_data(swap_data); - } - }); - inner_instruction_events.extend(events); - } - } - } - inner_instruction_events - }; - let (r1, r2_1, r2_2) = tokio::join!(task1, task2_1, task2_2); - instruction_events.extend(r1); - instruction_events.extend(r2_1); - inner_instruction_events.extend(r2_2); + Ok(()) + } - if !instruction_events.is_empty() && !inner_instruction_events.is_empty() { - for instruction_event in &mut instruction_events { - for inner_instruction_event in &inner_instruction_events { - if instruction_event.id() == inner_instruction_event.id() { - if instruction_event.instruction_inner_index().is_none() - && inner_instruction_event.instruction_inner_index().is_some() - { - if inner_instruction_event.instruction_outer_index() - == instruction_event.instruction_outer_index() - { - instruction_event.merge(inner_instruction_event.as_ref()); - break; - } - } else if instruction_event.instruction_inner_index().is_some() - && inner_instruction_event.instruction_inner_index().is_some() - { - if instruction_event.instruction_outer_index() - == inner_instruction_event.instruction_outer_index() - { - if inner_instruction_event - .instruction_inner_index() - .unwrap_or_default() - > instruction_event - .instruction_inner_index() - .unwrap_or_default() - { - instruction_event.merge(inner_instruction_event.as_ref()); - break; - } + async fn parse_encoded_confirmed_transaction_with_status_meta( + &self, + signature: Signature, + transaction: EncodedConfirmedTransactionWithStatusMeta, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { + let versioned_tx = match transaction.transaction.transaction.decode() { + Some(tx) => tx, + None => { + println!("Failed to decode transaction"); + return Ok(()); + } + }; + let mut inner_instructions_vec: Vec = Vec::new(); + if let Some(meta) = &transaction.transaction.meta { + // 从meta中获取inner_instructions，处理OptionSerializer类型 + if let solana_transaction_status::option_serializer::OptionSerializer::Some( + ui_inner_insts, + ) = &meta.inner_instructions + { + // 将UiInnerInstructions转换为InnerInstructions + for ui_inner in ui_inner_insts { + let mut converted_instructions = Vec::new(); + + // 转换每个UiInstruction为InnerInstruction + for ui_instruction in &ui_inner.instructions { + if let UiInstruction::Compiled(ui_compiled) = ui_instruction { + // 解码base58编码的data + if let Ok(data) = bs58::decode(&ui_compiled.data).into_vec() { + let compiled_instruction = CompiledInstruction { + program_id_index: ui_compiled.program_id_index, + accounts: ui_compiled.accounts.clone(), + data, + }; + + let inner_instruction = InnerInstruction { + instruction: compiled_instruction, + stack_height: ui_compiled.stack_height, + }; + + converted_instructions.push(inner_instruction); } } } - } - } - } - let result = self.process_events(instruction_events, bot_wallet); - - Ok(result) - } - - fn process_events( - &self, - mut events: Vec>, - bot_wallet: Option, - ) -> Vec> { - let mut dev_address = vec![]; - let mut bonk_dev_address = None; - for event in &mut events { - if let Some(token_info) = event.as_any().downcast_ref::() { - dev_address.push(token_info.user); - if token_info.creator != Pubkey::default() && token_info.creator != token_info.user - { - dev_address.push(token_info.creator); - } - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() - { - if dev_address.contains(&trade_info.user) - || dev_address.contains(&trade_info.creator) - { - trade_info.is_dev_create_token_trade = true; - } else if Some(trade_info.user) == bot_wallet { - trade_info.is_bot = true; - } else { - trade_info.is_dev_create_token_trade = false; - } - if trade_info.metadata.swap_data.is_some() { - trade_info.metadata.swap_data.as_mut().unwrap().from_amount = - if trade_info.is_buy { - trade_info.sol_amount - } else { - trade_info.token_amount - }; - trade_info.metadata.swap_data.as_mut().unwrap().to_amount = if trade_info.is_buy - { - trade_info.token_amount - } else { - trade_info.sol_amount + let inner_instructions = InnerInstructions { + index: ui_inner.index, + instructions: converted_instructions, }; - } - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { - if trade_info.metadata.swap_data.is_some() { - trade_info.metadata.swap_data.as_mut().unwrap().from_amount = - trade_info.user_quote_amount_in; - trade_info.metadata.swap_data.as_mut().unwrap().to_amount = - trade_info.base_amount_out; - } - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() - { - if trade_info.metadata.swap_data.is_some() { - trade_info.metadata.swap_data.as_mut().unwrap().from_amount = - trade_info.base_amount_in; - trade_info.metadata.swap_data.as_mut().unwrap().to_amount = - trade_info.user_quote_amount_out; - } - } else if let Some(pool_info) = event.as_any().downcast_ref::() { - bonk_dev_address = Some(pool_info.creator); - } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { - if Some(trade_info.payer) == bonk_dev_address { - trade_info.is_dev_create_token_trade = true; - } else if Some(trade_info.payer) == bot_wallet { - trade_info.is_bot = true; - } else { - trade_info.is_dev_create_token_trade = false; + + inner_instructions_vec.push(inner_instructions); } } - event.clear_id(); + } + let inner_instructions: &[InnerInstructions] = &inner_instructions_vec; + + let meta = transaction.transaction.meta; + let mut address_table_lookups: Vec = vec![]; + if let Some(meta) = meta { + if let solana_transaction_status::option_serializer::OptionSerializer::Some( + loaded_addresses, + ) = &meta.loaded_addresses + { + address_table_lookups + .reserve(loaded_addresses.writable.len() + loaded_addresses.readonly.len()); + address_table_lookups.extend( + loaded_addresses + .writable + .iter() + .filter_map(|s| s.parse::().ok()) + .chain( + loaded_addresses + .readonly + .iter() + .filter_map(|s| s.parse::().ok()), + ), + ); + } + } + let mut accounts = Vec::with_capacity( + versioned_tx.message.static_account_keys().len() + address_table_lookups.len(), + ); + accounts.extend_from_slice(versioned_tx.message.static_account_keys()); + accounts.extend(address_table_lookups); + // 使用 Arc 包装共享数据，避免不必要的克隆 + let accounts_arc = Arc::new(accounts); + let inner_instructions_arc = Arc::new(inner_instructions); + + let slot = transaction.slot; + let block_time = transaction.block_time.map(|t| Timestamp { seconds: t as i64, nanos: 0 }); + let program_received_time_us = chrono::Utc::now().timestamp_micros(); + let bot_wallet = None; + let transaction_index = None; + // 解析指令事件 + self.parse_instruction_events_from_versioned_transaction( + &versioned_tx, + signature, + Some(slot), + block_time, + program_received_time_us, + &accounts_arc, + &inner_instructions_arc, + bot_wallet, + transaction_index, + callback.clone(), + ) + .await?; + + // 解析嵌套指令事件 + for inner_instruction in inner_instructions_arc.iter() { + for (index, instruction) in inner_instruction.instructions.iter().enumerate() { + self.parse_instruction( + &instruction.instruction, + &accounts_arc, + signature, + Some(slot), + block_time, + program_received_time_us, + inner_instruction.index as i64, + Some(index as i64), + bot_wallet, + transaction_index, + Some(&inner_instruction), + callback.clone(), + ) + .await?; + } } - events + Ok(()) } async fn parse_inner_instruction( @@ -511,9 +475,9 @@ pub trait EventParser: Send + Sync { program_received_time_us: i64, outer_index: i64, inner_index: Option, - bot_wallet: Option, transaction_index: Option, - ) -> Result>> { + config: &GenericEventParseConfig, + ) -> anyhow::Result>> { let slot = slot.unwrap_or(0); let events = self.parse_events_from_inner_instruction( instruction, @@ -523,8 +487,8 @@ pub trait EventParser: Send + Sync { program_received_time_us, outer_index, inner_index, - bot_wallet, transaction_index, + config, ); Ok(events) } @@ -542,9 +506,11 @@ pub trait EventParser: Send + Sync { inner_index: Option, bot_wallet: Option, transaction_index: Option, - ) -> Result>> { + inner_instructions: Option<&InnerInstructions>, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let slot = slot.unwrap_or(0); - let events = self.parse_events_from_instruction( + self.parse_events_from_instruction( instruction, accounts, signature, @@ -555,8 +521,9 @@ pub trait EventParser: Send + Sync { inner_index, bot_wallet, transaction_index, - ); - Ok(events) + inner_instructions, + callback, + ) } /// 检查是否应该处理此程序ID @@ -596,7 +563,7 @@ pub type InstructionEventParser = /// 通用事件解析器基类 pub struct GenericEventParser { pub program_ids: Vec, - pub inner_instruction_configs: HashMap, Vec>, + // pub inner_instruction_configs: HashMap, Vec>, pub instruction_configs: HashMap, Vec>, } @@ -604,23 +571,16 @@ impl GenericEventParser { /// 创建新的通用事件解析器 pub fn new(program_ids: Vec, configs: Vec) -> Self { // 预分配容量，避免动态扩容 - let mut inner_instruction_configs = HashMap::with_capacity(configs.len()); let mut instruction_configs = HashMap::with_capacity(configs.len()); for config in configs { - if config.inner_instruction_discriminator.len() > 0 { - inner_instruction_configs - .entry(config.inner_instruction_discriminator.to_vec()) - .or_insert_with(Vec::new) - .push(config.clone()); - } instruction_configs .entry(config.instruction_discriminator.to_vec()) .or_insert_with(Vec::new) .push(config.clone()); } - Self { program_ids, inner_instruction_configs, instruction_configs } + Self { program_ids, instruction_configs } } /// 通用的内联指令解析方法 @@ -701,12 +661,7 @@ impl GenericEventParser { #[async_trait::async_trait] impl EventParser for GenericEventParser { - fn inner_instruction_configs(&self) -> HashMap, Vec> { - // 返回引用而非克隆，减少内存分配 - self.inner_instruction_configs.clone() - } fn instruction_configs(&self) -> HashMap, Vec> { - // 返回引用而非克隆，减少内存分配 self.instruction_configs.clone() } /// 从内联指令中解析事件数据 @@ -720,32 +675,26 @@ impl EventParser for GenericEventParser { program_received_time_us: i64, outer_index: i64, inner_index: Option, - bot_wallet: Option, transaction_index: Option, + config: &GenericEventParseConfig, ) -> Vec> { if inner_instruction.data.len() < 16 { return Vec::new(); } let data = &inner_instruction.data[16..]; let mut events = Vec::new(); - for (disc, configs) in &self.inner_instruction_configs { - if data == disc { - for config in configs { - if let Some(event) = self.parse_inner_instruction_event( - config, - data, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - transaction_index, - ) { - events.push(event); - } - } - } + if let Some(event) = self.parse_inner_instruction_event( + config, + data, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + ) { + events.push(event); } events } @@ -764,12 +713,13 @@ impl EventParser for GenericEventParser { inner_index: Option, bot_wallet: Option, transaction_index: Option, - ) -> Vec> { + inner_instructions: Option<&InnerInstructions>, + callback: Arc Fn(&'a Box) + Send + Sync>, + ) -> anyhow::Result<()> { let program_id = accounts[instruction.program_id_index as usize]; if !self.should_handle(&program_id) { - return Vec::new(); + return Ok(()); } - let mut events = Vec::new(); for (disc, configs) in &self.instruction_configs { if instruction.data.len() < disc.len() { continue; @@ -781,14 +731,13 @@ impl EventParser for GenericEventParser { if !validate_account_indices(&instruction.accounts, accounts.len()) { continue; } - let account_pubkeys: Vec = instruction.accounts.iter().map(|&idx| accounts[idx as usize]).collect(); for config in configs { if config.program_id != program_id { continue; } - if let Some(event) = self.parse_instruction_event( + if let Some(mut event) = self.parse_instruction_event( config, data, &account_pubkeys, @@ -800,13 +749,53 @@ impl EventParser for GenericEventParser { inner_index, transaction_index, ) { - events.push(event); + let mut inner_instruction_event: Option> = None; + if inner_instructions.is_some() { + // 解析对应的内部 log 执行 + for inner_instruction in inner_instructions.unwrap().instructions.iter() + { + let result = self.parse_events_from_inner_instruction( + &inner_instruction.instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + config, + ); + if result.len() > 0 { + inner_instruction_event = Some(result[0].clone()); + } + // 解析swap数据 + let swap_data = parse_swap_data_from_next_instructions( + &*event, + inner_instructions.unwrap(), + inner_index.unwrap_or(-1_i64) as i8, + &accounts, + ); + if let Some(swap_data) = swap_data { + event.set_swap_data(swap_data); + } + } + } + // 合并事件 + if let Some(inner_instruction_event) = inner_instruction_event { + event.merge(&*inner_instruction_event); + } + // 设置处理时间 + event.set_program_handle_time_consuming_us( + chrono::Utc::now().timestamp_micros() - program_received_time_us, + ); + event = process_event(event, bot_wallet); + callback(&event); + break; } } } } - - events + Ok(()) } fn should_handle(&self, program_id: &Pubkey) -> bool { @@ -817,3 +806,54 @@ impl EventParser for GenericEventParser { self.program_ids.clone() } } + +fn process_event( + mut event: Box, + bot_wallet: Option, +) -> Box { + update_slot(event.slot()); + if let Some(token_info) = event.as_any().downcast_ref::() { + add_dev_address(token_info.user); + if token_info.creator != Pubkey::default() && token_info.creator != token_info.user { + add_dev_address(token_info.creator); + } + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if is_dev_address(&trade_info.user) || is_dev_address(&trade_info.creator) { + trade_info.is_dev_create_token_trade = true; + } else if Some(trade_info.user) == bot_wallet { + trade_info.is_bot = true; + } else { + trade_info.is_dev_create_token_trade = false; + } + if trade_info.metadata.swap_data.is_some() { + trade_info.metadata.swap_data.as_mut().unwrap().from_amount = + if trade_info.is_buy { trade_info.sol_amount } else { trade_info.token_amount }; + trade_info.metadata.swap_data.as_mut().unwrap().to_amount = + if trade_info.is_buy { trade_info.token_amount } else { trade_info.sol_amount }; + } + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if trade_info.metadata.swap_data.is_some() { + trade_info.metadata.swap_data.as_mut().unwrap().from_amount = + trade_info.user_quote_amount_in; + trade_info.metadata.swap_data.as_mut().unwrap().to_amount = trade_info.base_amount_out; + } + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if trade_info.metadata.swap_data.is_some() { + trade_info.metadata.swap_data.as_mut().unwrap().from_amount = trade_info.base_amount_in; + trade_info.metadata.swap_data.as_mut().unwrap().to_amount = + trade_info.user_quote_amount_out; + } + } else if let Some(pool_info) = event.as_any().downcast_ref::() { + add_bonk_dev_address(pool_info.creator); + } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { + if is_bonk_dev_address(&trade_info.payer) { + trade_info.is_dev_create_token_trade = true; + } else if Some(trade_info.payer) == bot_wallet { + trade_info.is_bot = true; + } else { + trade_info.is_dev_create_token_trade = false; + } + } + event.clear_id(); + event +} diff --git a/src/streaming/event_parser/protocols/mutil/parser.rs b/src/streaming/event_parser/protocols/mutil/parser.rs index c8b58a0..e64edfd 100755 --- a/src/streaming/event_parser/protocols/mutil/parser.rs +++ b/src/streaming/event_parser/protocols/mutil/parser.rs @@ -18,24 +18,6 @@ impl MutilEventParser { for protocol in protocols { let parse = EventParserFactory::create_parser(protocol); - // Merge inner_instruction_configs, append configurations to existing Vec - for (key, configs) in parse.inner_instruction_configs() { - let filtered_configs: Vec = configs - .into_iter() - .filter(|config| { - event_type_filter - .as_ref() - .map(|filter| filter.include.contains(&config.event_type)) - .unwrap_or(true) - }) - .collect(); - inner - .inner_instruction_configs - .entry(key) - .or_insert_with(Vec::new) - .extend(filtered_configs); - } - // Merge instruction_configs, append configurations to existing Vec for (key, configs) in parse.instruction_configs() { let filtered_configs: Vec = configs diff --git a/src/streaming/grpc/mod.rs b/src/streaming/grpc/mod.rs index 9c012e6..0d91713 100644 --- a/src/streaming/grpc/mod.rs +++ b/src/streaming/grpc/mod.rs @@ -1,17 +1,15 @@ // gRPC 相关模块 pub mod connection; -pub mod stream_handler; pub mod subscription; pub mod types; // 重新导出主要类型 pub use connection::*; -pub use stream_handler::*; pub use subscription::*; pub use types::*; // 从公用模块重新导出 pub use crate::streaming::common::{ - BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, MetricsManager, - PerformanceMetrics, StreamClientConfig as ClientConfig, + BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsManager, PerformanceMetrics, + StreamClientConfig as ClientConfig, }; diff --git a/src/streaming/grpc/stream_handler.rs b/src/streaming/grpc/stream_handler.rs deleted file mode 100644 index 43dbb77..0000000 --- a/src/streaming/grpc/stream_handler.rs +++ /dev/null @@ -1,100 +0,0 @@ -use chrono::Local; -use futures::{channel::mpsc, sink::Sink, SinkExt}; -use solana_sdk::pubkey::Pubkey; -use yellowstone_grpc_proto::geyser::{ - subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, SubscribeUpdate, -}; - -use super::types::{BlockMetaPretty, EventPretty, TransactionPretty}; -use crate::common::AnyResult; -use crate::streaming::common::EventProcessor; -use crate::streaming::grpc::AccountPretty; - -/// 流消息处理器 -pub struct StreamHandler; - -impl StreamHandler { - /// 处理单个流消息 - pub async fn handle_stream_message( - msg: SubscribeUpdate, - subscribe_tx: &mut (impl Sink + Unpin), - event_processor: EventProcessor, - bot_wallet: Option, - ) -> AnyResult<()> { - let created_at = msg.created_at; - match msg.update_oneof { - Some(UpdateOneof::Account(account)) => { - let account_pretty = AccountPretty::from(account); - log::debug!("Received account: {:?}", account_pretty); - event_processor - .process_grpc_event_transaction_with_metrics( - EventPretty::Account(account_pretty), - bot_wallet, - ) - .await?; - } - Some(UpdateOneof::BlockMeta(sut)) => { - let block_meta_pretty = BlockMetaPretty::from((sut, created_at)); - log::debug!("Received block meta: {:?}", block_meta_pretty); - event_processor - .process_grpc_event_transaction_with_metrics( - EventPretty::BlockMeta(block_meta_pretty), - bot_wallet, - ) - .await?; - } - Some(UpdateOneof::Transaction(sut)) => { - let transaction_pretty = TransactionPretty::from((sut, created_at)); - log::debug!( - "Received transaction: {} at slot {}", - transaction_pretty.signature, - transaction_pretty.slot - ); - event_processor - .process_grpc_event_transaction_with_metrics( - EventPretty::Transaction(transaction_pretty), - bot_wallet, - ) - .await?; - } - Some(UpdateOneof::Ping(_)) => { - subscribe_tx - .send(SubscribeRequest { - ping: Some(SubscribeRequestPing { id: 1 }), - ..Default::default() - }) - .await?; - log::debug!("service is ping: {}", Local::now()); - } - Some(UpdateOneof::Pong(_)) => { - log::debug!("service is pong: {}", Local::now()); - } - _ => { - log::debug!("Received other message type"); - } - } - Ok(()) - } - - pub async fn handle_stream_system_message( - msg: SubscribeUpdate, - subscribe_tx: &mut (impl Sink + Unpin), - ) -> AnyResult> { - let created_at = msg.created_at; - let event_pretty = match msg.update_oneof { - Some(UpdateOneof::Transaction(sut)) => Some(TransactionPretty::from((sut, created_at))), - Some(UpdateOneof::Ping(_)) => { - subscribe_tx - .send(SubscribeRequest { - ping: Some(SubscribeRequestPing { id: 1 }), - ..Default::default() - }) - .await?; - None - } - Some(UpdateOneof::Pong(_)) => None, - _ => None, - }; - Ok(event_pretty.map(|e| EventPretty::Transaction(e))) - } -} diff --git a/src/streaming/shred/connection.rs b/src/streaming/shred/connection.rs index 44e7e4a..9c6bbe3 100644 --- a/src/streaming/shred/connection.rs +++ b/src/streaming/shred/connection.rs @@ -41,16 +41,33 @@ impl ShredStreamGrpc { }) } - /// 创建高性能客户端（适合高并发场景） - pub async fn new_high_performance(endpoint: String) -> AnyResult { - Self::new_with_config(endpoint, StreamClientConfig::high_performance()).await + /// Creates a new ShredStreamClient with high-throughput configuration. + /// + /// This is a convenience method that creates a client optimized for high-concurrency scenarios + /// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()` + /// for detailed configuration information. + pub async fn new_high_throughput(endpoint: String) -> AnyResult { + Self::new_with_config(endpoint, StreamClientConfig::high_throughput()).await } - /// 创建低延迟客户端（适合实时场景） + /// Creates a new ShredStreamClient with low-latency configuration. + /// + /// This is a convenience method that creates a client optimized for real-time scenarios + /// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()` + /// for detailed configuration information. pub async fn new_low_latency(endpoint: String) -> AnyResult { Self::new_with_config(endpoint, StreamClientConfig::low_latency()).await } + /// Creates a new ShredStreamClient with asynchronous processing configuration. + /// + /// This is a convenience method that creates a client optimized for high-volume scenarios + /// with balanced throughput and reliability. See `StreamClientConfig::async_processing()` + /// for detailed configuration information. + pub async fn new_async_processing(endpoint: String) -> AnyResult { + Self::new_with_config(endpoint, StreamClientConfig::async_processing()).await + } + /// 获取当前配置 pub fn get_config(&self) -> &StreamClientConfig { &self.config diff --git a/src/streaming/shred/mod.rs b/src/streaming/shred/mod.rs index c2aa2d9..1358fe9 100644 --- a/src/streaming/shred/mod.rs +++ b/src/streaming/shred/mod.rs @@ -8,6 +8,6 @@ pub use types::*; // 从公用模块重新导出 pub use crate::streaming::common::{ - BackpressureConfig, BackpressureStrategy, BatchConfig, ConnectionConfig, MetricsEventType, - MetricsManager, PerformanceMetrics, StreamClientConfig, + BackpressureConfig, BackpressureStrategy, ConnectionConfig, MetricsEventType, MetricsManager, + PerformanceMetrics, StreamClientConfig, }; diff --git a/src/streaming/shred/types.rs b/src/streaming/shred/types.rs index 888622d..d10094e 100644 --- a/src/streaming/shred/types.rs +++ b/src/streaming/shred/types.rs @@ -5,16 +5,16 @@ use solana_sdk::transaction::VersionedTransaction; pub struct TransactionWithSlot { pub transaction: VersionedTransaction, pub slot: u64, + pub program_received_time_us: i64, } impl TransactionWithSlot { /// 创建新的带槽位的交易 - pub fn new(transaction: VersionedTransaction, slot: u64) -> Self { - Self { transaction, slot } - } - - /// 获取交易签名 - pub fn signature(&self) -> String { - self.transaction.signatures[0].to_string() + pub fn new( + transaction: VersionedTransaction, + slot: u64, + program_received_time_us: i64, + ) -> Self { + Self { transaction, slot, program_received_time_us } } } diff --git a/src/streaming/shred_stream.rs b/src/streaming/shred_stream.rs index 79cda1b..1ec59af 100755 --- a/src/streaming/shred_stream.rs +++ b/src/streaming/shred_stream.rs @@ -42,7 +42,6 @@ impl ShredStreamGrpc { protocols, event_type_filter, self.config.backpressure.clone(), - self.config.batch.clone(), Some(Arc::new(callback)), ); @@ -58,18 +57,24 @@ impl ShredStreamGrpc { if let Ok(entries) = bincode::deserialize::>(&msg.entries) { for entry in entries { for transaction in entry.transactions { - let transaction_with_slot = - TransactionWithSlot::new(transaction.clone(), msg.slot); - if let Err(e) = event_processor_clone - .process_shred_transaction_immediate( - transaction_with_slot, - bot_wallet, - ) - .await - { - error!("Error handling message: {e:?}"); - break; - } + let transaction_with_slot = TransactionWithSlot::new( + transaction.clone(), + msg.slot, + chrono::Utc::now().timestamp_micros(), + ); + // 异步执行，不阻塞主流，使用带背压控制的方法 + let processor_clone = event_processor_clone.clone(); + tokio::spawn(async move { + if let Err(e) = processor_clone + .process_shred_transaction_with_metrics( + transaction_with_slot, + bot_wallet, + ) + .await + { + error!("Error handling message: {e:?}"); + } + }); } } } diff --git a/src/streaming/yellowstone_grpc.rs b/src/streaming/yellowstone_grpc.rs index db928d9..6c12a41 100644 --- a/src/streaming/yellowstone_grpc.rs +++ b/src/streaming/yellowstone_grpc.rs @@ -1,9 +1,11 @@ -use futures::StreamExt; +use chrono::Local; +use futures::{SinkExt, StreamExt}; use log::error; use solana_sdk::pubkey::Pubkey; use std::sync::{Arc, RwLock}; use tokio::sync::Mutex; -use yellowstone_grpc_proto::geyser::CommitmentLevel; +use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestPing}; use crate::common::AnyResult; use crate::streaming::common::{ @@ -11,7 +13,9 @@ use crate::streaming::common::{ }; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::{Protocol, UnifiedEvent}; -use crate::streaming::grpc::{StreamHandler, SubscriptionManager}; +use crate::streaming::grpc::{ + AccountPretty, BlockMetaPretty, EventPretty, SubscriptionManager, TransactionPretty, +}; /// 交易过滤器 pub struct TransactionFilter { @@ -73,20 +77,31 @@ impl YellowstoneGrpc { }) } - /// 创建高性能客户端 - pub fn new_high_performance(endpoint: String, x_token: Option) -> AnyResult { - Self::new_with_config(endpoint, x_token, StreamClientConfig::high_performance()) + /// Creates a new YellowstoneGrpcClient with high-throughput configuration. + /// + /// This is a convenience method that creates a client optimized for high-concurrency scenarios + /// where throughput is prioritized over latency. See `StreamClientConfig::high_throughput()` + /// for detailed configuration information. + pub fn new_high_throughput(endpoint: String, x_token: Option) -> AnyResult { + Self::new_with_config(endpoint, x_token, StreamClientConfig::high_throughput()) } - /// 创建低延迟客户端 + /// Creates a new YellowstoneGrpcClient with low-latency configuration. + /// + /// This is a convenience method that creates a client optimized for real-time scenarios + /// where latency is prioritized over throughput. See `StreamClientConfig::low_latency()` + /// for detailed configuration information. pub fn new_low_latency(endpoint: String, x_token: Option) -> AnyResult { Self::new_with_config(endpoint, x_token, StreamClientConfig::low_latency()) } - /// 创建即时处理客户端 - pub fn new_immediate(endpoint: String, x_token: Option) -> AnyResult { - let mut config = StreamClientConfig::low_latency(); - config.enable_metrics = false; + /// Creates a new YellowstoneGrpcClient with asynchronous processing configuration. + /// + /// This is a convenience method that creates a client optimized for high-volume scenarios + /// with balanced throughput and reliability. See `StreamClientConfig::async_processing()` + /// for detailed configuration information. + pub fn new_async_processing(endpoint: String, x_token: Option) -> AnyResult { + let config = StreamClientConfig::async_processing(); Self::new_with_config(endpoint, x_token, config) } @@ -171,35 +186,98 @@ impl YellowstoneGrpc { ); // 订阅事件 - let (mut subscribe_tx, mut stream) = self + let (subscribe_tx, mut stream) = self .subscription_manager .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) .await?; + // 用 Arc> 包装 subscribe_tx 以支持多线程共享 + let subscribe_tx = Arc::new(Mutex::new(subscribe_tx)); + // 启动流处理任务 let mut event_processor = self.event_processor.clone(); event_processor.set_protocols_and_event_type_filter( protocols, event_type_filter, self.config.backpressure.clone(), - self.config.batch.clone(), Some(Arc::new(callback)), ); + let event_processor = Arc::new(event_processor); let stream_handle = tokio::spawn(async move { while let Some(message) = stream.next().await { match message { Ok(msg) => { - if let Err(e) = StreamHandler::handle_stream_message( - msg, - &mut subscribe_tx, - event_processor.clone(), - bot_wallet, - ) - .await - { - error!("Error handling message: {e:?}"); - break; - } + // 不阻塞地处理消息，使用 tokio::spawn 实现并发 + let event_processor_ref = Arc::clone(&event_processor); + let subscribe_tx_ref = Arc::clone(&subscribe_tx); + tokio::spawn(async move { + let created_at = msg.created_at; + match msg.update_oneof { + Some(UpdateOneof::Account(account)) => { + let account_pretty = AccountPretty::from(account); + log::debug!("Received account: {:?}", account_pretty); + if let Err(e) = event_processor_ref + .process_grpc_event_transaction_with_metrics( + EventPretty::Account(account_pretty), + bot_wallet, + ) + .await + { + error!("Error processing account event: {e:?}"); + } + } + Some(UpdateOneof::BlockMeta(sut)) => { + let block_meta_pretty = + BlockMetaPretty::from((sut, created_at)); + log::debug!("Received block meta: {:?}", block_meta_pretty); + if let Err(e) = event_processor_ref + .process_grpc_event_transaction_with_metrics( + EventPretty::BlockMeta(block_meta_pretty), + bot_wallet, + ) + .await + { + error!("Error processing block meta event: {e:?}"); + } + } + Some(UpdateOneof::Transaction(sut)) => { + let transaction_pretty = + TransactionPretty::from((sut, created_at)); + log::debug!( + "Received transaction: {} at slot {}", + transaction_pretty.signature, + transaction_pretty.slot + ); + if let Err(e) = event_processor_ref + .process_grpc_event_transaction_with_metrics( + EventPretty::Transaction(transaction_pretty), + bot_wallet, + ) + .await + { + error!("Error processing transaction event: {e:?}"); + } + } + Some(UpdateOneof::Ping(_)) => { + // 只在需要时获取锁，并立即释放 + if let Ok(mut tx_guard) = subscribe_tx_ref.try_lock() { + let _ = tx_guard + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: 1 }), + ..Default::default() + }) + .await; + } + log::debug!("service is ping: {}", Local::now()); + } + Some(UpdateOneof::Pong(_)) => { + log::debug!("service is pong: {}", Local::now()); + } + _ => { + log::debug!("Received other message type"); + } + } + }); } Err(error) => { error!("Stream error: {error:?}"); diff --git a/src/streaming/yellowstone_sub_system.rs b/src/streaming/yellowstone_sub_system.rs index bccd417..91c991f 100755 --- a/src/streaming/yellowstone_sub_system.rs +++ b/src/streaming/yellowstone_sub_system.rs @@ -1,15 +1,18 @@ use crate::{ common::AnyResult, streaming::{ - grpc::{EventPretty, StreamHandler}, + grpc::{EventPretty, TransactionPretty}, yellowstone_grpc::YellowstoneGrpc, }, }; -use futures::StreamExt; +use futures::{SinkExt, StreamExt}; use log::error; use solana_program::pubkey; use solana_sdk::{pubkey::Pubkey, transaction::VersionedTransaction}; use solana_transaction_status::TransactionWithStatusMeta; +use yellowstone_grpc_proto::geyser::{ + subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestPing, +}; const SYSTEM_PROGRAM_ID: Pubkey = pubkey!("11111111111111111111111111111111"); @@ -34,7 +37,7 @@ impl YellowstoneGrpc { account_exclude: Option>, ) -> AnyResult<()> where - F: Fn(SystemEvent) + Send + Sync + 'static, + F: Fn(SystemEvent) + Send + Sync + Clone + 'static, { let addrs = vec![SYSTEM_PROGRAM_ID.to_string()]; let account_include = account_include.unwrap_or_default(); @@ -56,16 +59,36 @@ impl YellowstoneGrpc { while let Some(message) = stream.next().await { match message { Ok(msg) => { - if let Ok(event_pretty) = - StreamHandler::handle_stream_system_message(msg, &mut subscribe_tx) - .await - { - if let Some(event_pretty) = event_pretty { - if let Err(e) = - Self::process_system_transaction(event_pretty, &*callback).await - { - error!("Error processing transaction: {e:?}"); - } + let created_at = msg.created_at; + match msg.update_oneof { + Some(UpdateOneof::Transaction(sut)) => { + let transaction_pretty = TransactionPretty::from((sut, created_at)); + let event_pretty = EventPretty::Transaction(transaction_pretty); + let callback_clone = callback.clone(); + tokio::spawn(async move { + if let Err(e) = Self::process_system_transaction( + event_pretty, + &*callback_clone, + ) + .await + { + error!("Error processing transaction: {e:?}"); + } + }); + } + Some(UpdateOneof::Ping(_)) => { + let _ = subscribe_tx + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: 1 }), + ..Default::default() + }) + .await; + } + Some(UpdateOneof::Pong(_)) => { + // Pong response, no action needed + } + _ => { + // Other message types, ignore for system subscription } } } From 74781e5cbe6f48877ea564e4ae9da6ec51282b16 Mon Sep 17 00:00:00 2001 From: ysq Date: Sun, 31 Aug 2025 22:18:18 +0800 Subject: [PATCH 7/7] perf: implement SIMD-accelerated event processing and optimize streaming performance - Add SIMD utilities for fast byte array comparison and discriminator matching - Optimize event processor with batch processing and memory pool - Refactor global state management with concurrent data structures - Remove deprecated batch processing module - Enhance metrics collection with reduced overhead - Improve parser efficiency across all protocol implementations - Add performance benchmarking dependencies (criterion, wide) - Update documentation and examples for new architecture Performance improvements: - SIMD-accelerated byte operations for instruction parsing - Concurrent HashMap (DashMap) for better multi-threading - Optimized memory allocation patterns - Reduced lock contention in event processing pipeline Breaking changes: Removed batch.rs module, updated parser interfaces --- .gitignore | 3 +- Cargo.toml | 5 + README.md | 32 +- README_CN.md | 34 +- examples/parse_tx_events.rs | 9 +- src/main.rs | 294 +++++++------- src/streaming/common/batch.rs | 131 ------ src/streaming/common/config.rs | 27 +- src/streaming/common/event_processor.rs | 373 ++++++++++-------- src/streaming/common/metrics.rs | 259 +++--------- src/streaming/common/mod.rs | 6 +- src/streaming/common/simd_utils.rs | 295 ++++++++++++++ src/streaming/event_parser/common/mod.rs | 14 +- src/streaming/event_parser/common/types.rs | 46 +-- .../event_parser/core/account_event_parser.rs | 36 +- .../event_parser/core/common_event_parser.rs | 4 +- .../event_parser/core/global_state.rs | 238 +++++++---- src/streaming/event_parser/core/traits.rs | 337 +++++++++++----- .../protocols/block/block_meta_event.rs | 5 +- .../event_parser/protocols/bonk/parser.rs | 28 -- .../event_parser/protocols/pumpfun/parser.rs | 22 -- .../event_parser/protocols/pumpswap/parser.rs | 55 --- .../protocols/raydium_amm_v4/parser.rs | 36 -- .../protocols/raydium_clmm/parser.rs | 24 -- .../protocols/raydium_cpmm/parser.rs | 18 - src/streaming/grpc/subscription.rs | 26 +- src/streaming/shred/connection.rs | 8 - src/streaming/shred_stream.rs | 23 +- src/streaming/yellowstone_grpc.rs | 149 ++++--- src/streaming/yellowstone_sub_system.rs | 19 +- 30 files changed, 1297 insertions(+), 1259 deletions(-) delete mode 100644 src/streaming/common/batch.rs create mode 100644 src/streaming/common/simd_utils.rs diff --git a/.gitignore b/.gitignore index c77fa9e..4433903 100755 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ Cargo.lock # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -.cargo/ \ No newline at end of file +.cargo/ +.claude/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index d3ed2ce..ce72227 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ thiserror = "2.0.11" async-trait = "0.1.86" lazy_static = "1.5.0" once_cell = "1.20.3" +dashmap = "6.0.1" prost = "0.13.5" prost-types = "0.13.5" num_enum = "0.7.3" @@ -66,3 +67,7 @@ env_logger = "0.11.8" crossbeam = "0.8.4" crossbeam-queue = "0.3.12" parking_lot = "0.12.1" +wide = "0.7" + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } \ No newline at end of file diff --git a/README.md b/README.md index a09432e..011c2a2 100755 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ A lightweight Rust library for real-time event streaming from Solana DEX trading 11. **Performance Monitoring**: Built-in performance metrics monitoring, including event processing speed, etc. 12. **Memory Optimization**: Object pooling and caching mechanisms to reduce memory allocations 13. **Flexible Configuration System**: Support for custom batch sizes, backpressure strategies, channel sizes, and other parameters -14. **Preset Configurations**: Provides high-throughput, low-latency, and async processing preset configurations optimized for different use cases -15. **Backpressure Handling**: Supports blocking, dropping, retrying, ordered, and other backpressure strategies +14. **Preset Configurations**: Provides high-throughput and low-latency preset configurations optimized for different use cases +15. **Backpressure Handling**: Supports blocking and dropping backpressure strategies 16. **Runtime Configuration Updates**: Supports dynamic configuration parameter updates at runtime 17. **Full Function Performance Monitoring**: All subscribe_events functions support performance monitoring, automatically collecting and reporting performance metrics 18. **Graceful Shutdown**: Support for programmatic stop() method for clean shutdown @@ -91,26 +91,10 @@ let shred = ShredStreamGrpc::new_low_latency(endpoint).await?; **Features:** - **Backpressure Strategy**: Block - ensures no data loss -- **Buffer Size**: 1 permit to minimize memory usage +- **Buffer Size**: 4000 permits for balanced throughput and latency - **Immediate Processing**: No buffering, processes events immediately - **Use Case**: Scenarios where every millisecond counts and you cannot afford to lose any events, such as trading applications or real-time monitoring -#### 3. Async Processing Configuration (`async_processing()`) - -Balances throughput and reliability: - -```rust -let config = StreamClientConfig::async_processing(); -// Or use convenience methods -let grpc = YellowstoneGrpc::new_async_processing(endpoint, token)?; -let shred = ShredStreamGrpc::new_async_processing(endpoint).await?; -``` - -**Features:** -- **Backpressure Strategy**: Async - non-blocking operation -- **Buffer Size**: 5,000 permits for steady flow -- **Fire-and-forget**: Async processing semantics -- **Use Case**: Scenarios where you need sustained high throughput with eventual consistency, such as data ingestion pipelines or event streaming applications ### Custom Configuration @@ -131,16 +115,6 @@ let config = StreamClientConfig { }; ``` -### Configuration Selection Guide - -| Scenario | Recommended Config | Reason | -|----------|-------------------|---------| -| Trading Bots | `low_latency()` | Need fastest response time, cannot lose trading signals | -| Data Analytics | `high_throughput()` | Need to process large amounts of historical data, can tolerate some data loss | -| Event Stream Processing | `async_processing()` | Balance performance and reliability, suitable for continuous processing | -| Real-time Monitoring | `low_latency()` | Need immediate response to anomalies | -| Bulk Data Ingestion | `high_throughput()` | Prioritize overall throughput | - ## Usage Examples ### Quick Start - Parse Transaction Events diff --git a/README_CN.md b/README_CN.md index cf4330a..e0c94dc 100644 --- a/README_CN.md +++ b/README_CN.md @@ -24,8 +24,8 @@ 11. **性能监控**: 内置性能指标监控，包括事件处理速度等 12. **内存优化**: 对象池和缓存机制减少内存分配 13. **灵活配置系统**: 支持自定义批处理大小、背压策略、通道大小等参数 -14. **预设配置**: 提供高吞吐量、低延迟、异步处理等预设配置，针对不同使用场景优化 -15. **背压处理**: 支持阻塞、丢弃、重试、有序等多种背压策略 +14. **预设配置**: 提供高吞吐量、低延迟等预设配置，针对不同使用场景优化 +15. **背压处理**: 支持阻塞、丢弃等背压策略 16. **运行时配置更新**: 支持在运行时动态更新配置参数 17. **全函数性能监控**: 所有subscribe_events函数都支持性能监控，自动收集和报告性能指标 18. **优雅关闭**: 支持编程式 stop() 方法进行干净的关闭 @@ -90,26 +90,10 @@ let shred = ShredStreamGrpc::new_low_latency(endpoint).await?; **特性：** - **背压策略**: Block（阻塞策略）- 确保不丢失任何数据 -- **缓冲区大小**: 1 个许可证，最小化内存使用 +- **缓冲区大小**: 4000 个许可证，平衡吞吐量和延迟 - **立即处理**: 不进行缓冲，立即处理事件 - **适用场景**: 每毫秒都很重要且不能丢失任何事件的场景，如交易应用或实时监控 -#### 3. 异步处理配置 (`async_processing()`) - -在吞吐量和可靠性之间取得平衡： - -```rust -let config = StreamClientConfig::async_processing(); -// 或者使用便捷方法 -let grpc = YellowstoneGrpc::new_async_processing(endpoint, token)?; -let shred = ShredStreamGrpc::new_async_processing(endpoint).await?; -``` - -**特性：** -- **背压策略**: Async（异步策略）- 非阻塞操作 -- **缓冲区大小**: 5,000 个许可证，保持稳定流量 -- **Fire-and-forget**: 异步处理语义 -- **适用场景**: 需要持续高吞吐量且可接受最终一致性的场景，如数据摄取管道或事件流应用 ### 自定义配置 @@ -130,16 +114,6 @@ let config = StreamClientConfig { }; ``` -### 配置选择指南 - -| 场景 | 推荐配置 | 原因 | -|------|----------|------| -| 交易机器人 | `low_latency()` | 需要最快响应时间，不能丢失交易信号 | -| 数据分析 | `high_throughput()` | 需要处理大量历史数据，可容忍部分数据丢失 | -| 事件流处理 | `async_processing()` | 平衡性能和可靠性，适合持续处理 | -| 实时监控 | `low_latency()` | 需要立即响应异常情况 | -| 批量数据摄取 | `high_throughput()` | 优先考虑整体吞吐量 | - ## 使用示例 ### 快速开始 - 解析交易事件 @@ -587,7 +561,7 @@ let event_type_filter = Some(EventTypeFilter { - **Yellowstone gRPC 客户端**: 针对 Solana 事件流优化 - **ShredStream 客户端**: 替代流实现 -- **异步处理**: 非阻塞事件处理 +- **高性能处理**: 优化的事件处理机制 ## 项目结构 diff --git a/examples/parse_tx_events.rs b/examples/parse_tx_events.rs index 40b9a02..5f54b8d 100644 --- a/examples/parse_tx_events.rs +++ b/examples/parse_tx_events.rs @@ -5,10 +5,6 @@ use solana_streamer_sdk::streaming::event_parser::UnifiedEvent; use solana_streamer_sdk::streaming::event_parser::{ protocols::MutilEventParser, EventParser, Protocol, }; -use solana_transaction_status::{InnerInstruction, InnerInstructions, UiInstruction}; - -use solana_sdk::bs58; -use solana_sdk::instruction::CompiledInstruction; use std::str::FromStr; use std::sync::Arc; @@ -16,7 +12,7 @@ use std::sync::Arc; #[tokio::main] async fn main() -> Result<()> { let signatures = vec![ - "4PsHYajH87x2zJPEGZczZtd2ksibuMCFPonC24jk5mTGZ46hzvjpzM5UZuLz9sRv79MkCBbtDqwJapGPTSkCFKoL", + "5sDWrTTkE69CNc6nrAX7SqPS7FiajJTg8TMog3Gve7KjVfrqYn8YZcX1kAoyKok976S4RTnK1EdCV8hRiDWg68Aj", ]; // Validate signature format let mut valid_signatures = Vec::new(); @@ -119,5 +115,8 @@ async fn get_single_transaction_details(signature_str: &str) -> Result<()> { } } + println!("Press Ctrl+C to exit example..."); + tokio::signal::ctrl_c().await?; + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 51fc068..d6ad5c3 100755 --- a/src/main.rs +++ b/src/main.rs @@ -53,7 +53,7 @@ use solana_streamer_sdk::{ async fn main() -> Result<(), Box> { println!("Starting Solana Streamer..."); test_grpc().await?; - // test_shreds().await?; + test_shreds().await?; Ok(()) } @@ -188,151 +188,151 @@ async fn test_shreds() -> Result<(), Box> { fn create_event_callback() -> impl Fn(Box) { |event: Box| { - // println!( - // "🎉 Event received! Type: {:?}, transaction_index: {:?}", - // event.event_type(), - // event.transaction_index() - // ); - // match_event!(event, { - // // -------------------------- block meta ----------------------- - // BlockMetaEvent => |e: BlockMetaEvent| { - // println!("BlockMetaEvent: {:?}", e.metadata.program_handle_time_consuming_us); - // }, - // // -------------------------- bonk ----------------------- - // BonkPoolCreateEvent => |e: BonkPoolCreateEvent| { - // // When using grpc, you can get block_time from each event - // println!("block_time: {:?}, block_time_ms: {:?}", e.metadata.block_time, e.metadata.block_time_ms); - // println!("BonkPoolCreateEvent: {:?}", e.base_mint_param.symbol); - // }, - // BonkTradeEvent => |e: BonkTradeEvent| { - // println!("BonkTradeEvent: {e:?}"); - // }, - // BonkMigrateToAmmEvent => |e: BonkMigrateToAmmEvent| { - // println!("BonkMigrateToAmmEvent: {e:?}"); - // }, - // BonkMigrateToCpswapEvent => |e: BonkMigrateToCpswapEvent| { - // println!("BonkMigrateToCpswapEvent: {e:?}"); - // }, - // // -------------------------- pumpfun ----------------------- - // PumpFunTradeEvent => |e: PumpFunTradeEvent| { - // println!("PumpFunTradeEvent: {e:?}"); - // }, - // PumpFunMigrateEvent => |e: PumpFunMigrateEvent| { - // println!("PumpFunMigrateEvent: {e:?}"); - // }, - // PumpFunCreateTokenEvent => |e: PumpFunCreateTokenEvent| { - // println!("PumpFunCreateTokenEvent: {e:?}"); - // }, - // // -------------------------- pumpswap ----------------------- - // PumpSwapBuyEvent => |e: PumpSwapBuyEvent| { - // println!("Buy event: {e:?}"); - // }, - // PumpSwapSellEvent => |e: PumpSwapSellEvent| { - // println!("Sell event: {e:?}"); - // }, - // PumpSwapCreatePoolEvent => |e: PumpSwapCreatePoolEvent| { - // println!("CreatePool event: {e:?}"); - // }, - // PumpSwapDepositEvent => |e: PumpSwapDepositEvent| { - // println!("Deposit event: {e:?}"); - // }, - // PumpSwapWithdrawEvent => |e: PumpSwapWithdrawEvent| { - // println!("Withdraw event: {e:?}"); - // }, - // // -------------------------- raydium_cpmm ----------------------- - // RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| { - // println!("RaydiumCpmmSwapEvent: {e:?}"); - // }, - // RaydiumCpmmDepositEvent => |e: RaydiumCpmmDepositEvent| { - // println!("RaydiumCpmmDepositEvent: {e:?}"); - // }, - // RaydiumCpmmInitializeEvent => |e: RaydiumCpmmInitializeEvent| { - // println!("RaydiumCpmmInitializeEvent: {e:?}"); - // }, - // RaydiumCpmmWithdrawEvent => |e: RaydiumCpmmWithdrawEvent| { - // println!("RaydiumCpmmWithdrawEvent: {e:?}"); - // }, - // // -------------------------- raydium_clmm ----------------------- - // RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| { - // println!("RaydiumClmmSwapEvent: {e:?}"); - // }, - // RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| { - // println!("RaydiumClmmSwapV2Event: {e:?}"); - // }, - // RaydiumClmmClosePositionEvent => |e: RaydiumClmmClosePositionEvent| { - // println!("RaydiumClmmClosePositionEvent: {e:?}"); - // }, - // RaydiumClmmDecreaseLiquidityV2Event => |e: RaydiumClmmDecreaseLiquidityV2Event| { - // println!("RaydiumClmmDecreaseLiquidityV2Event: {e:?}"); - // }, - // RaydiumClmmCreatePoolEvent => |e: RaydiumClmmCreatePoolEvent| { - // println!("RaydiumClmmCreatePoolEvent: {e:?}"); - // }, - // RaydiumClmmIncreaseLiquidityV2Event => |e: RaydiumClmmIncreaseLiquidityV2Event| { - // println!("RaydiumClmmIncreaseLiquidityV2Event: {e:?}"); - // }, - // RaydiumClmmOpenPositionWithToken22NftEvent => |e: RaydiumClmmOpenPositionWithToken22NftEvent| { - // println!("RaydiumClmmOpenPositionWithToken22NftEvent: {e:?}"); - // }, - // RaydiumClmmOpenPositionV2Event => |e: RaydiumClmmOpenPositionV2Event| { - // println!("RaydiumClmmOpenPositionV2Event: {e:?}"); - // }, - // // -------------------------- raydium_amm_v4 ----------------------- - // RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| { - // println!("RaydiumAmmV4SwapEvent: {e:?}"); - // }, - // RaydiumAmmV4DepositEvent => |e: RaydiumAmmV4DepositEvent| { - // println!("RaydiumAmmV4DepositEvent: {e:?}"); - // }, - // RaydiumAmmV4Initialize2Event => |e: RaydiumAmmV4Initialize2Event| { - // println!("RaydiumAmmV4Initialize2Event: {e:?}"); - // }, - // RaydiumAmmV4WithdrawEvent => |e: RaydiumAmmV4WithdrawEvent| { - // println!("RaydiumAmmV4WithdrawEvent: {e:?}"); - // }, - // RaydiumAmmV4WithdrawPnlEvent => |e: RaydiumAmmV4WithdrawPnlEvent| { - // println!("RaydiumAmmV4WithdrawPnlEvent: {e:?}"); - // }, - // // -------------------------- account ----------------------- - // BonkPoolStateAccountEvent => |e: BonkPoolStateAccountEvent| { - // println!("BonkPoolStateAccountEvent: {e:?}"); - // }, - // BonkGlobalConfigAccountEvent => |e: BonkGlobalConfigAccountEvent| { - // println!("BonkGlobalConfigAccountEvent: {e:?}"); - // }, - // BonkPlatformConfigAccountEvent => |e: BonkPlatformConfigAccountEvent| { - // println!("BonkPlatformConfigAccountEvent: {e:?}"); - // }, - // PumpSwapGlobalConfigAccountEvent => |e: PumpSwapGlobalConfigAccountEvent| { - // println!("PumpSwapGlobalConfigAccountEvent: {e:?}"); - // }, - // PumpSwapPoolAccountEvent => |e: PumpSwapPoolAccountEvent| { - // println!("PumpSwapPoolAccountEvent: {e:?}"); - // }, - // PumpFunBondingCurveAccountEvent => |e: PumpFunBondingCurveAccountEvent| { - // println!("PumpFunBondingCurveAccountEvent: {e:?}"); - // }, - // PumpFunGlobalAccountEvent => |e: PumpFunGlobalAccountEvent| { - // println!("PumpFunGlobalAccountEvent: {e:?}"); - // }, - // RaydiumAmmV4AmmInfoAccountEvent => |e: RaydiumAmmV4AmmInfoAccountEvent| { - // println!("RaydiumAmmV4AmmInfoAccountEvent: {e:?}"); - // }, - // RaydiumClmmAmmConfigAccountEvent => |e: RaydiumClmmAmmConfigAccountEvent| { - // println!("RaydiumClmmAmmConfigAccountEvent: {e:?}"); - // }, - // RaydiumClmmPoolStateAccountEvent => |e: RaydiumClmmPoolStateAccountEvent| { - // println!("RaydiumClmmPoolStateAccountEvent: {e:?}"); - // }, - // RaydiumClmmTickArrayStateAccountEvent => |e: RaydiumClmmTickArrayStateAccountEvent| { - // println!("RaydiumClmmTickArrayStateAccountEvent: {e:?}"); - // }, - // RaydiumCpmmAmmConfigAccountEvent => |e: RaydiumCpmmAmmConfigAccountEvent| { - // println!("RaydiumCpmmAmmConfigAccountEvent: {e:?}"); - // }, - // RaydiumCpmmPoolStateAccountEvent => |e: RaydiumCpmmPoolStateAccountEvent| { - // println!("RaydiumCpmmPoolStateAccountEvent: {e:?}"); - // }, - // }); + println!( + "🎉 Event received! Type: {:?}, transaction_index: {:?}", + event.event_type(), + event.transaction_index() + ); + match_event!(event, { + // -------------------------- block meta ----------------------- + BlockMetaEvent => |e: BlockMetaEvent| { + println!("BlockMetaEvent: {:?}", e.metadata.program_handle_time_consuming_us); + }, + // -------------------------- bonk ----------------------- + BonkPoolCreateEvent => |e: BonkPoolCreateEvent| { + // When using grpc, you can get block_time from each event + println!("block_time: {:?}, block_time_ms: {:?}", e.metadata.block_time, e.metadata.block_time_ms); + println!("BonkPoolCreateEvent: {:?}", e.base_mint_param.symbol); + }, + BonkTradeEvent => |e: BonkTradeEvent| { + println!("BonkTradeEvent: {e:?}"); + }, + BonkMigrateToAmmEvent => |e: BonkMigrateToAmmEvent| { + println!("BonkMigrateToAmmEvent: {e:?}"); + }, + BonkMigrateToCpswapEvent => |e: BonkMigrateToCpswapEvent| { + println!("BonkMigrateToCpswapEvent: {e:?}"); + }, + // -------------------------- pumpfun ----------------------- + PumpFunTradeEvent => |e: PumpFunTradeEvent| { + println!("PumpFunTradeEvent: {e:?}"); + }, + PumpFunMigrateEvent => |e: PumpFunMigrateEvent| { + println!("PumpFunMigrateEvent: {e:?}"); + }, + PumpFunCreateTokenEvent => |e: PumpFunCreateTokenEvent| { + println!("PumpFunCreateTokenEvent: {e:?}"); + }, + // -------------------------- pumpswap ----------------------- + PumpSwapBuyEvent => |e: PumpSwapBuyEvent| { + println!("Buy event: {e:?}"); + }, + PumpSwapSellEvent => |e: PumpSwapSellEvent| { + println!("Sell event: {e:?}"); + }, + PumpSwapCreatePoolEvent => |e: PumpSwapCreatePoolEvent| { + println!("CreatePool event: {e:?}"); + }, + PumpSwapDepositEvent => |e: PumpSwapDepositEvent| { + println!("Deposit event: {e:?}"); + }, + PumpSwapWithdrawEvent => |e: PumpSwapWithdrawEvent| { + println!("Withdraw event: {e:?}"); + }, + // -------------------------- raydium_cpmm ----------------------- + RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| { + println!("RaydiumCpmmSwapEvent: {e:?}"); + }, + RaydiumCpmmDepositEvent => |e: RaydiumCpmmDepositEvent| { + println!("RaydiumCpmmDepositEvent: {e:?}"); + }, + RaydiumCpmmInitializeEvent => |e: RaydiumCpmmInitializeEvent| { + println!("RaydiumCpmmInitializeEvent: {e:?}"); + }, + RaydiumCpmmWithdrawEvent => |e: RaydiumCpmmWithdrawEvent| { + println!("RaydiumCpmmWithdrawEvent: {e:?}"); + }, + // -------------------------- raydium_clmm ----------------------- + RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| { + println!("RaydiumClmmSwapEvent: {e:?}"); + }, + RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| { + println!("RaydiumClmmSwapV2Event: {e:?}"); + }, + RaydiumClmmClosePositionEvent => |e: RaydiumClmmClosePositionEvent| { + println!("RaydiumClmmClosePositionEvent: {e:?}"); + }, + RaydiumClmmDecreaseLiquidityV2Event => |e: RaydiumClmmDecreaseLiquidityV2Event| { + println!("RaydiumClmmDecreaseLiquidityV2Event: {e:?}"); + }, + RaydiumClmmCreatePoolEvent => |e: RaydiumClmmCreatePoolEvent| { + println!("RaydiumClmmCreatePoolEvent: {e:?}"); + }, + RaydiumClmmIncreaseLiquidityV2Event => |e: RaydiumClmmIncreaseLiquidityV2Event| { + println!("RaydiumClmmIncreaseLiquidityV2Event: {e:?}"); + }, + RaydiumClmmOpenPositionWithToken22NftEvent => |e: RaydiumClmmOpenPositionWithToken22NftEvent| { + println!("RaydiumClmmOpenPositionWithToken22NftEvent: {e:?}"); + }, + RaydiumClmmOpenPositionV2Event => |e: RaydiumClmmOpenPositionV2Event| { + println!("RaydiumClmmOpenPositionV2Event: {e:?}"); + }, + // -------------------------- raydium_amm_v4 ----------------------- + RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| { + println!("RaydiumAmmV4SwapEvent: {e:?}"); + }, + RaydiumAmmV4DepositEvent => |e: RaydiumAmmV4DepositEvent| { + println!("RaydiumAmmV4DepositEvent: {e:?}"); + }, + RaydiumAmmV4Initialize2Event => |e: RaydiumAmmV4Initialize2Event| { + println!("RaydiumAmmV4Initialize2Event: {e:?}"); + }, + RaydiumAmmV4WithdrawEvent => |e: RaydiumAmmV4WithdrawEvent| { + println!("RaydiumAmmV4WithdrawEvent: {e:?}"); + }, + RaydiumAmmV4WithdrawPnlEvent => |e: RaydiumAmmV4WithdrawPnlEvent| { + println!("RaydiumAmmV4WithdrawPnlEvent: {e:?}"); + }, + // -------------------------- account ----------------------- + BonkPoolStateAccountEvent => |e: BonkPoolStateAccountEvent| { + println!("BonkPoolStateAccountEvent: {e:?}"); + }, + BonkGlobalConfigAccountEvent => |e: BonkGlobalConfigAccountEvent| { + println!("BonkGlobalConfigAccountEvent: {e:?}"); + }, + BonkPlatformConfigAccountEvent => |e: BonkPlatformConfigAccountEvent| { + println!("BonkPlatformConfigAccountEvent: {e:?}"); + }, + PumpSwapGlobalConfigAccountEvent => |e: PumpSwapGlobalConfigAccountEvent| { + println!("PumpSwapGlobalConfigAccountEvent: {e:?}"); + }, + PumpSwapPoolAccountEvent => |e: PumpSwapPoolAccountEvent| { + println!("PumpSwapPoolAccountEvent: {e:?}"); + }, + PumpFunBondingCurveAccountEvent => |e: PumpFunBondingCurveAccountEvent| { + println!("PumpFunBondingCurveAccountEvent: {e:?}"); + }, + PumpFunGlobalAccountEvent => |e: PumpFunGlobalAccountEvent| { + println!("PumpFunGlobalAccountEvent: {e:?}"); + }, + RaydiumAmmV4AmmInfoAccountEvent => |e: RaydiumAmmV4AmmInfoAccountEvent| { + println!("RaydiumAmmV4AmmInfoAccountEvent: {e:?}"); + }, + RaydiumClmmAmmConfigAccountEvent => |e: RaydiumClmmAmmConfigAccountEvent| { + println!("RaydiumClmmAmmConfigAccountEvent: {e:?}"); + }, + RaydiumClmmPoolStateAccountEvent => |e: RaydiumClmmPoolStateAccountEvent| { + println!("RaydiumClmmPoolStateAccountEvent: {e:?}"); + }, + RaydiumClmmTickArrayStateAccountEvent => |e: RaydiumClmmTickArrayStateAccountEvent| { + println!("RaydiumClmmTickArrayStateAccountEvent: {e:?}"); + }, + RaydiumCpmmAmmConfigAccountEvent => |e: RaydiumCpmmAmmConfigAccountEvent| { + println!("RaydiumCpmmAmmConfigAccountEvent: {e:?}"); + }, + RaydiumCpmmPoolStateAccountEvent => |e: RaydiumCpmmPoolStateAccountEvent| { + println!("RaydiumCpmmPoolStateAccountEvent: {e:?}"); + }, + }); } } diff --git a/src/streaming/common/batch.rs b/src/streaming/common/batch.rs deleted file mode 100644 index f9c092c..0000000 --- a/src/streaming/common/batch.rs +++ /dev/null @@ -1,131 +0,0 @@ -use crate::streaming::event_parser::UnifiedEvent; - -/// 通用批处理事件收集器 -pub struct EventBatchProcessor -where - F: FnMut(Vec>) + Send + Sync + 'static, -{ - pub(crate) callback: F, - batch: Vec>, - batch_size: usize, - timeout_ms: u64, - last_flush_time: std::time::Instant, -} - -impl EventBatchProcessor -where - F: FnMut(Vec>) + Send + Sync + 'static, -{ - /// 创建新的批处理器 - pub fn new(callback: F, batch_size: usize, timeout_ms: u64) -> Self { - Self { - callback, - batch: Vec::with_capacity(batch_size), - batch_size, - timeout_ms, - last_flush_time: std::time::Instant::now(), - } - } - - /// 添加事件到批次 - pub fn add_event(&mut self, event: Box) { - log::debug!("Adding event to batch: {} (type: {:?})", event.id(), event.event_type()); - self.batch.push(event); - - // 检查是否需要刷新批次 - if self.batch.len() >= self.batch_size || self.should_flush_by_timeout() { - log::debug!("Flushing batch: size={}, timeout={}", self.batch.len(), self.should_flush_by_timeout()); - self.flush(); - } - } - - /// 强制刷新当前批次 - pub fn flush(&mut self) { - if !self.batch.is_empty() { - let events = std::mem::replace(&mut self.batch, Vec::with_capacity(self.batch_size)); - log::debug!("Flushing {} events from batch processor", events.len()); - - // 添加调试信息（仅在debug模式下） - if log::log_enabled!(log::Level::Debug) { - for (i, event) in events.iter().enumerate() { - log::debug!("Event {}: Type={:?}, ID={}", i, event.event_type(), event.id()); - } - } - - // 执行回调并捕获可能的错误 - log::debug!("Executing batch callback with {} events", events.len()); - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - (self.callback)(events); - })) { - Ok(_) => { - log::debug!("Batch callback executed successfully"); - } - Err(e) => { - log::error!("Batch callback panicked: {:?}", e); - } - } - - self.last_flush_time = std::time::Instant::now(); - } else { - log::debug!("No events to flush"); - } - } - - /// 获取当前批次大小 - pub fn current_batch_size(&self) -> usize { - self.batch.len() - } - - /// 检查是否应该基于超时刷新 - fn should_flush_by_timeout(&self) -> bool { - self.last_flush_time.elapsed().as_millis() >= self.timeout_ms as u128 - } - - /// 检查批次是否已满 - pub fn is_batch_full(&self) -> bool { - self.batch.len() >= self.batch_size - } - - /// 检查是否需要刷新（大小或超时） - pub fn should_flush(&self) -> bool { - self.is_batch_full() || self.should_flush_by_timeout() - } -} - -/// 简单的事件批处理器，用于将单个事件回调转换为批量回调 -pub struct SimpleEventBatchProcessor -where - F: Fn(Box) + Send + Sync + 'static, -{ - callback: F, -} - -impl SimpleEventBatchProcessor -where - F: Fn(Box) + Send + Sync + 'static, -{ - pub fn new(callback: F) -> Self { - Self { callback } - } - - /// 将批量事件拆分为单个事件处理 - pub fn process_batch(&self, events: Vec>) { - for event in events { - (self.callback)(event); - } - } -} - -/// 批处理器包装器，用于将单个事件回调适配为批量处理 -pub fn create_batch_callback_adapter( - single_event_callback: F, -) -> impl FnMut(Vec>) + Send + Sync + 'static -where - F: Fn(Box) + Send + Sync + 'static, -{ - move |events: Vec>| { - for event in events { - single_event_callback(event); - } - } -} diff --git a/src/streaming/common/config.rs b/src/streaming/common/config.rs index 0c76687..ce21ec8 100644 --- a/src/streaming/common/config.rs +++ b/src/streaming/common/config.rs @@ -7,8 +7,6 @@ pub enum BackpressureStrategy { Block, /// Drop messages Drop, - /// Execute asynchronously (don't wait for completion) - Async, } impl Default for BackpressureStrategy { @@ -87,7 +85,7 @@ impl StreamClientConfig { Self { connection: ConnectionConfig::default(), backpressure: BackpressureConfig { - permits: 5000, + permits: 20000, strategy: BackpressureStrategy::Drop, }, enable_metrics: false, @@ -99,35 +97,16 @@ impl StreamClientConfig { /// This configuration prioritizes latency over throughput by: /// - Processing events immediately without buffering /// - Implementing a blocking backpressure strategy to ensure no data loss - /// - Setting minimal permits (1) to minimize memory usage + /// - Setting optimal permits (4000) for balanced throughput and latency /// /// Ideal for scenarios where every millisecond counts and you cannot /// afford to lose any events, such as trading applications or real-time monitoring. pub fn low_latency() -> Self { Self { connection: ConnectionConfig::default(), - backpressure: BackpressureConfig { permits: 1, strategy: BackpressureStrategy::Block }, + backpressure: BackpressureConfig { permits: 4000, strategy: BackpressureStrategy::Block }, enable_metrics: false, } } - /// Creates an asynchronous processing configuration optimized for high-volume scenarios. - /// - /// This configuration balances throughput and reliability by: - /// - Implementing an async backpressure strategy for non-blocking operation - /// - Setting a balanced permit buffer (5,000) for steady flow - /// - /// Ideal for scenarios where you need sustained high throughput with - /// fire-and-forget semantics, such as data ingestion pipelines or - /// event streaming applications where some eventual consistency is acceptable. - pub fn async_processing() -> Self { - Self { - connection: ConnectionConfig::default(), - backpressure: BackpressureConfig { - permits: 5000, - strategy: BackpressureStrategy::Async, - }, - enable_metrics: false, - } - } } diff --git a/src/streaming/common/event_processor.rs b/src/streaming/common/event_processor.rs index 571a6b5..7ae82f7 100644 --- a/src/streaming/common/event_processor.rs +++ b/src/streaming/common/event_processor.rs @@ -1,17 +1,19 @@ +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Instant; +use crossbeam_queue::SegQueue; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::Signature; -use tokio::sync::Semaphore; use crate::common::AnyResult; +use crate::streaming::common::BackpressureStrategy; use crate::streaming::common::{ MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig, }; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::core::account_event_parser::AccountEventParser; use crate::streaming::event_parser::core::common_event_parser::CommonEventParser; +use crate::streaming::event_parser::core::traits::get_high_perf_clock; use crate::streaming::event_parser::EventParser; use crate::streaming::event_parser::{ core::traits::UnifiedEvent, protocols::mutil::parser::MutilEventParser, Protocol, @@ -20,7 +22,7 @@ use crate::streaming::grpc::{BackpressureConfig, EventPretty}; use crate::streaming::shred::TransactionWithSlot; use once_cell::sync::OnceCell; -/// Event processor +/// High-performance Event processor using SegQueue for all strategies pub struct EventProcessor { pub(crate) metrics_manager: MetricsManager, pub(crate) config: ClientConfig, @@ -29,15 +31,27 @@ pub struct EventProcessor { pub(crate) event_type_filter: Option, pub(crate) callback: Option) + Send + Sync>>, pub(crate) backpressure_config: BackpressureConfig, - /// Backpressure semaphore for controlling concurrent processing count - pub(crate) backpressure_semaphore: Arc, + /// High-performance lockfree queue for gRPC events + pub(crate) grpc_queue: Arc)>>, + /// High-performance lockfree queue for shred events + pub(crate) shred_queue: Arc)>>, + /// Fast O(1) counter for Drop strategy (avoids expensive SegQueue::len()) + pub(crate) grpc_pending_count: Arc, + pub(crate) shred_pending_count: Arc, + /// Processing thread control + pub(crate) processing_shutdown: Arc, } impl EventProcessor { - /// Create a new event processor + /// Create a new high-performance event processor pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self { let backpressure_config = config.backpressure.clone(); - let backpressure_semaphore = Arc::new(Semaphore::new(backpressure_config.permits)); + let grpc_queue = Arc::new(SegQueue::new()); + let shred_queue = Arc::new(SegQueue::new()); + let grpc_pending_count = Arc::new(AtomicUsize::new(0)); + let shred_pending_count = Arc::new(AtomicUsize::new(0)); + let processing_shutdown = Arc::new(AtomicBool::new(false)); + Self { metrics_manager, config, @@ -46,7 +60,11 @@ impl EventProcessor { event_type_filter: None, backpressure_config, callback: None, - backpressure_semaphore, + grpc_queue, + shred_queue, + grpc_pending_count, + shred_pending_count, + processing_shutdown, } } @@ -57,94 +75,100 @@ impl EventProcessor { backpressure_config: BackpressureConfig, callback: Option) + Send + Sync>>, ) { - self.protocols = protocols.clone(); - self.event_type_filter = event_type_filter.clone(); - // Recreate semaphore if backpressure configuration changes - if self.backpressure_config.permits != backpressure_config.permits { - self.backpressure_semaphore = Arc::new(Semaphore::new(backpressure_config.permits)); - } + self.protocols = protocols; + self.event_type_filter = event_type_filter; + + // Check if Block processing thread should be started (before moving backpressure_config) + let should_start_block_processing = true; + // matches!(backpressure_config.strategy, BackpressureStrategy::Block); + self.backpressure_config = backpressure_config; self.callback = callback; - self.parser_cache - .get_or_init(|| Arc::new(MutilEventParser::new(protocols, event_type_filter))); + // Use stored values to initialize parser_cache + let protocols_ref = &self.protocols; + let event_type_filter_ref = self.event_type_filter.as_ref(); + self.parser_cache.get_or_init(|| { + Arc::new(MutilEventParser::new(protocols_ref.clone(), event_type_filter_ref.cloned())) + }); + + // Start Block processing thread if using Block strategy + if should_start_block_processing { + self.start_block_processing_thread(); + } } pub fn get_parser(&self) -> Arc { self.parser_cache.get().unwrap().clone() } + /// Create adapter callback + fn create_adapter_callback(&self) -> Arc) + Send + Sync> { + let callback = self.callback.clone().unwrap(); + let metrics_manager = self.metrics_manager.clone(); + + Arc::new(move |event: Box| { + let processing_time_us = event.program_handle_time_consuming_us() as f64; + callback(event); + metrics_manager.update_metrics(MetricsEventType::Transaction, 1, processing_time_us); + }) + } + pub async fn process_grpc_event_transaction_with_metrics( &self, event_pretty: EventPretty, bot_wallet: Option, ) -> AnyResult<()> { - // Backpressure control logic - let backpressure_start = Instant::now(); - let result = self.apply_backpressure_control(event_pretty, bot_wallet).await; - let backpressure_duration = backpressure_start.elapsed(); - - // Record backpressure-related metrics - self.metrics_manager.record_backpressure_metrics( - backpressure_duration, - result.is_ok(), - self.backpressure_semaphore.available_permits(), - ); - - result + self.apply_backpressure_control(event_pretty, bot_wallet).await } - /// Apply backpressure control strategy + /// Apply backpressure control strategy async fn apply_backpressure_control( &self, event_pretty: EventPretty, bot_wallet: Option, ) -> AnyResult<()> { - use crate::streaming::common::BackpressureStrategy; - match self.backpressure_config.strategy { BackpressureStrategy::Block => { - // Blocking strategy: acquire semaphore permit - let _permit = - self.backpressure_semaphore.acquire().await.map_err(|e| { - anyhow::anyhow!("Failed to acquire backpressure permit: {}", e) - })?; - self.process_grpc_event_transaction(event_pretty, bot_wallet).await - } - BackpressureStrategy::Drop => { - // Drop strategy: try to acquire permit, drop if failed - match self.backpressure_semaphore.try_acquire() { - Ok(_permit) => { - let result = - self.process_grpc_event_transaction(event_pretty, bot_wallet).await; - result - } - Err(_) => { - // Record dropped event - self.metrics_manager.increment_dropped_events(); - Ok(()) + // Block strategy: async wait if queue is full (backpressure control) + loop { + let current_pending = self.grpc_pending_count.load(Ordering::Relaxed); + if current_pending < self.backpressure_config.permits { + self.grpc_queue.push((event_pretty, bot_wallet)); + self.grpc_pending_count.fetch_add(1, Ordering::Relaxed); + break; } + // Async yield to avoid blocking gRPC data source + tokio::task::yield_now().await; } - } - BackpressureStrategy::Async => { - // Async strategy: process asynchronously regardless of permits - self.spawn_async_processing(event_pretty, bot_wallet).await; Ok(()) } - } - } - - /// Process event asynchronously (without waiting for semaphore permit) - async fn spawn_async_processing(&self, event_pretty: EventPretty, bot_wallet: Option) { - let processor = self.clone(); - - tokio::spawn(async move { - // Async strategy: no semaphore control, allow unlimited concurrency - // Execute actual event processing directly - if let Err(e) = processor.process_grpc_event_transaction(event_pretty, bot_wallet).await - { - log::error!("Error in async event processing: {}", e); + BackpressureStrategy::Drop => { + // Drop strategy: Use O(1) atomic counter instead of expensive O(n) len() + // If pending count >= permits, DROP the event immediately + let current_pending = self.grpc_pending_count.load(Ordering::Relaxed); + if current_pending >= self.backpressure_config.permits { + self.metrics_manager.increment_dropped_events(); + Ok(()) + } else { + self.grpc_pending_count.fetch_add(1, Ordering::Relaxed); + let processor = self.clone(); + tokio::spawn(async move { + match processor + .process_grpc_event_transaction(event_pretty, bot_wallet) + .await + { + Ok(_) => { + processor.grpc_pending_count.fetch_sub(1, Ordering::Relaxed); + } + Err(e) => { + log::error!("Error in async gRPC processing: {}", e); + } + } + }); + Ok(()) + } } - }); + } } async fn process_grpc_event_transaction( @@ -158,21 +182,15 @@ impl EventProcessor { match event_pretty { EventPretty::Account(account_pretty) => { self.metrics_manager.add_account_process_count(); - let signature = account_pretty.signature; let account_event = AccountEventParser::parse_account_event( - self.protocols.clone(), + &self.protocols, account_pretty, - self.event_type_filter.clone(), + self.event_type_filter.as_ref(), ); if let Some(event) = account_event { let processing_time_us = event.program_handle_time_consuming_us() as f64; self.invoke_callback(event); - self.update_metrics( - MetricsEventType::Account, - 1, - processing_time_us, - Some(signature), - ); + self.update_metrics(MetricsEventType::Account, 1, processing_time_us); } } EventPretty::Transaction(transaction_pretty) => { @@ -185,18 +203,7 @@ impl EventProcessor { let transaction_index = transaction_pretty.transaction_index; // Use cache to get parser let parser = self.get_parser(); - let callback = self.callback.clone().unwrap(); - let metrics_manager = self.metrics_manager.clone(); - let adapter_callback = Arc::new(move |event: Box| { - let processing_time_us = event.program_handle_time_consuming_us() as f64; - callback(event); - metrics_manager.update_metrics( - MetricsEventType::Transaction, - 1, - processing_time_us, - Some(signature), - ); - }); + let adapter_callback = self.create_adapter_callback(); parser .parse_transaction_owned( tx, @@ -224,7 +231,7 @@ impl EventProcessor { ); let processing_time_us = block_meta_event.program_handle_time_consuming_us() as f64; self.invoke_callback(block_meta_event); - self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us, None); + self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us); } } @@ -253,18 +260,7 @@ impl EventProcessor { bot_wallet: Option, ) -> AnyResult<()> { // Backpressure control logic - let backpressure_start = Instant::now(); - let result = self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await; - let backpressure_duration = backpressure_start.elapsed(); - - // Record backpressure-related metrics - self.metrics_manager.record_backpressure_metrics( - backpressure_duration, - result.is_ok(), - self.backpressure_semaphore.available_permits(), - ); - - result + self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await } /// Apply shred backpressure control strategy @@ -273,57 +269,47 @@ impl EventProcessor { transaction_with_slot: TransactionWithSlot, bot_wallet: Option, ) -> AnyResult<()> { - use crate::streaming::common::BackpressureStrategy; - match self.backpressure_config.strategy { BackpressureStrategy::Block => { - // Blocking strategy: acquire semaphore permit - let _permit = - self.backpressure_semaphore.acquire().await.map_err(|e| { - anyhow::anyhow!("Failed to acquire backpressure permit: {}", e) - })?; - self.process_shred_transaction(transaction_with_slot, bot_wallet).await - } - BackpressureStrategy::Drop => { - // Drop strategy: try to acquire permit, drop if failed - match self.backpressure_semaphore.try_acquire() { - Ok(_permit) => { - let result = - self.process_shred_transaction(transaction_with_slot, bot_wallet).await; - result - } - Err(_) => { - // Record dropped event - self.metrics_manager.increment_dropped_events(); - Ok(()) + // Block strategy: async wait if queue is full (backpressure control) + loop { + let current_pending = self.shred_pending_count.load(Ordering::Relaxed); + if current_pending < self.backpressure_config.permits { + self.shred_queue.push((transaction_with_slot, bot_wallet)); + self.shred_pending_count.fetch_add(1, Ordering::Relaxed); + break; } + // Async yield to avoid blocking shred data source + tokio::task::yield_now().await; } - } - BackpressureStrategy::Async => { - // Async strategy: process asynchronously regardless of permits - self.spawn_async_shred_processing(transaction_with_slot, bot_wallet).await; Ok(()) } - } - } - - /// Process shred event asynchronously (without waiting for semaphore permit) - async fn spawn_async_shred_processing( - &self, - transaction_with_slot: TransactionWithSlot, - bot_wallet: Option, - ) { - let processor = self.clone(); - - tokio::spawn(async move { - // Async strategy: no semaphore control, allow unlimited concurrency - // Execute actual event processing directly - if let Err(e) = - processor.process_shred_transaction(transaction_with_slot, bot_wallet).await - { - log::error!("Error in async shred event processing: {}", e); + BackpressureStrategy::Drop => { + // Drop strategy: Use O(1) atomic counter instead of expensive O(n) len() + let current_pending = self.shred_pending_count.load(Ordering::Relaxed); + if current_pending >= self.backpressure_config.permits { + self.metrics_manager.increment_dropped_events(); + Ok(()) + } else { + self.shred_pending_count.fetch_add(1, Ordering::Relaxed); + let processor = self.clone(); + tokio::spawn(async move { + match processor + .process_shred_transaction(transaction_with_slot, bot_wallet) + .await + { + Ok(_) => { + processor.shred_pending_count.fetch_sub(1, Ordering::Relaxed); + } + Err(e) => { + log::error!("Error in async shred processing: {}", e); + } + } + }); + Ok(()) + } } - }); + } } pub async fn process_shred_transaction( @@ -342,20 +328,7 @@ impl EventProcessor { let program_received_time_us = transaction_with_slot.program_received_time_us; // Use cache to get parser let parser = self.get_parser(); - let callback = self.callback.clone().unwrap(); - let metrics_manager = self.metrics_manager.clone(); - - let adapter_callback = Arc::new(move |event: Box| { - let processing_time_us = event.program_handle_time_consuming_us() as f64; - callback(event); - metrics_manager.update_metrics( - MetricsEventType::Transaction, - 1, - processing_time_us, - Some(signature), - ); - }); - + let adapter_callback = self.create_adapter_callback(); parser .parse_versioned_transaction_owned( tx, @@ -373,14 +346,70 @@ impl EventProcessor { Ok(()) } - fn update_metrics( - &self, - ty: MetricsEventType, - count: u64, - time_us: f64, - signature: Option, - ) { - self.metrics_manager.update_metrics(ty, count, time_us, signature); + fn update_metrics(&self, ty: MetricsEventType, count: u64, time_us: f64) { + self.metrics_manager.update_metrics(ty, count, time_us); + } + + /// Start dedicated processing threads for all strategies + fn start_block_processing_thread(&self) { + // Reset shutdown flag + self.processing_shutdown.store(false, Ordering::Relaxed); + + let grpc_queue = Arc::clone(&self.grpc_queue); + let shred_queue = Arc::clone(&self.shred_queue); + let grpc_pending_count = Arc::clone(&self.grpc_pending_count); + let shred_pending_count = Arc::clone(&self.shred_pending_count); + let shutdown_flag = Arc::clone(&self.processing_shutdown); + let shutdown_flag_clone = Arc::clone(&self.processing_shutdown); + let processor = self.clone(); + let processor_clone = self.clone(); + // 1. 专用线程 + 2. Busy-wait + 4. 无锁处理 + std::thread::spawn(move || { + // 创建blocking runtime for async processing + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + while !shutdown_flag.load(Ordering::Relaxed) { + if let Some((event_pretty, bot_wallet)) = grpc_queue.pop() { + // Decrement pending counter when consuming from queue + grpc_pending_count.fetch_sub(1, Ordering::Relaxed); + // Process event in blocking runtime + if let Err(e) = rt.block_on( + processor.process_grpc_event_transaction(event_pretty, bot_wallet), + ) { + println!("Error processing gRPC event: {}", e); + } + } else { + // 2. 优化忙等待: 使用轻量级休眠减少CPU占用 + std::thread::yield_now(); + } + } + }); + + // Shred处理也使用相同的低延迟优化 + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + + while !shutdown_flag_clone.load(Ordering::Relaxed) { + if let Some((transaction_with_slot, bot_wallet)) = shred_queue.pop() { + // Decrement pending counter when consuming from queue + shred_pending_count.fetch_sub(1, Ordering::Relaxed); + // Process transaction in blocking runtime + if let Err(e) = rt.block_on( + processor_clone + .process_shred_transaction(transaction_with_slot, bot_wallet), + ) { + log::error!("Error processing shred transaction: {}", e); + } + } else { + // 优化忙等待: 使用轻量级休眠减少CPU占用 + std::thread::yield_now(); + } + } + }); + } + + /// Stop processing threads + pub fn stop_processing(&self) { + self.processing_shutdown.store(true, Ordering::Relaxed); } } @@ -395,7 +424,11 @@ impl Clone for EventProcessor { event_type_filter: self.event_type_filter.clone(), backpressure_config: self.backpressure_config.clone(), callback: self.callback.clone(), - backpressure_semaphore: self.backpressure_semaphore.clone(), + grpc_queue: self.grpc_queue.clone(), + shred_queue: self.shred_queue.clone(), + grpc_pending_count: self.grpc_pending_count.clone(), + shred_pending_count: self.shred_pending_count.clone(), + processing_shutdown: self.processing_shutdown.clone(), } } } diff --git a/src/streaming/common/metrics.rs b/src/streaming/common/metrics.rs index dd02697..402f909 100644 --- a/src/streaming/common/metrics.rs +++ b/src/streaming/common/metrics.rs @@ -1,11 +1,9 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; -use solana_sdk::signature::Signature; - use super::constants::*; -/// 事件类型枚举 +/// Event type enumeration #[derive(Debug, Clone, Copy)] pub enum EventType { Transaction = 0, @@ -13,7 +11,7 @@ pub enum EventType { BlockMeta = 2, } -/// 兼容性别名 +/// Compatibility alias pub type MetricsEventType = EventType; impl EventType { @@ -30,18 +28,18 @@ impl EventType { } } - // 兼容性常量 + // Compatibility constants pub const TX: EventType = EventType::Transaction; } -/// 高性能原子事件指标 +/// High-performance atomic event metrics #[derive(Debug)] struct AtomicEventMetrics { process_count: AtomicU64, events_processed: AtomicU64, events_in_window: AtomicU64, window_start_nanos: AtomicU64, - events_per_second_bits: AtomicU64, // f64 的位表示 + events_per_second_bits: AtomicU64, // Bit representation of f64 } impl AtomicEventMetrics { @@ -55,20 +53,20 @@ impl AtomicEventMetrics { } } - /// 原子地增加处理计数 + /// Atomically increment process count #[inline] fn add_process_count(&self) { self.process_count.fetch_add(1, Ordering::Relaxed); } - /// 原子地增加事件处理数量 + /// Atomically increment event processing count #[inline] fn add_events_processed(&self, count: u64) { self.events_processed.fetch_add(count, Ordering::Relaxed); self.events_in_window.fetch_add(count, Ordering::Relaxed); } - /// 获取当前计数（非阻塞） + /// Get current count (non-blocking) #[inline] fn get_counts(&self) -> (u64, u64, u64) { ( @@ -78,19 +76,19 @@ impl AtomicEventMetrics { ) } - /// 原子地更新每秒事件数 + /// Atomically update events per second #[inline] fn update_events_per_second(&self, eps: f64) { self.events_per_second_bits.store(eps.to_bits(), Ordering::Relaxed); } - /// 获取每秒事件数 + /// Get events per second #[inline] fn get_events_per_second(&self) -> f64 { f64::from_bits(self.events_per_second_bits.load(Ordering::Relaxed)) } - /// 重置窗口计数 + /// Reset window count #[inline] fn reset_window(&self, new_start_nanos: u64) { self.events_in_window.store(0, Ordering::Relaxed); @@ -103,13 +101,14 @@ impl AtomicEventMetrics { } } -/// 高性能原子处理时间统计 +/// High-performance atomic processing time statistics #[derive(Debug)] struct AtomicProcessingTimeStats { min_time_bits: AtomicU64, max_time_bits: AtomicU64, - max_time_timestamp_nanos: AtomicU64, // 最大值更新时间戳（纳秒） - total_time_us: AtomicU64, // 存储微秒的整数部分 + min_time_timestamp_nanos: AtomicU64, // Timestamp of min value update (nanoseconds) + max_time_timestamp_nanos: AtomicU64, // Timestamp of max value update (nanoseconds) + total_time_us: AtomicU64, // Store integer part of microseconds total_events: AtomicU64, } @@ -122,13 +121,14 @@ impl AtomicProcessingTimeStats { Self { min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()), max_time_bits: AtomicU64::new(0), + min_time_timestamp_nanos: AtomicU64::new(now_nanos), max_time_timestamp_nanos: AtomicU64::new(now_nanos), total_time_us: AtomicU64::new(0), total_events: AtomicU64::new(0), } } - /// 原子地更新处理时间统计 + /// Atomically update processing time statistics #[inline] fn update(&self, time_us: f64, event_count: u64) { let time_bits = time_us.to_bits(); @@ -136,8 +136,20 @@ impl AtomicProcessingTimeStats { std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() as u64; - // 更新最小值（使用 compare_exchange_weak 循环） + // Update minimum value, check time difference and reset if over 10 seconds let mut current_min = self.min_time_bits.load(Ordering::Relaxed); + let min_timestamp = self.min_time_timestamp_nanos.load(Ordering::Relaxed); + + // Check if min value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds) + let min_time_diff_nanos = now_nanos.saturating_sub(min_timestamp); + if min_time_diff_nanos > 10_000_000_000 { + // Over 10 seconds, reset min value + self.min_time_bits.store(f64::INFINITY.to_bits(), Ordering::Relaxed); + self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + current_min = f64::INFINITY.to_bits(); + } + + // If current time is less than min value, update min value and timestamp while time_bits < current_min { match self.min_time_bits.compare_exchange_weak( current_min, @@ -145,25 +157,29 @@ impl AtomicProcessingTimeStats { Ordering::Relaxed, Ordering::Relaxed, ) { - Ok(_) => break, + Ok(_) => { + // Successfully updated min value, also update timestamp + self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); + break; + } Err(x) => current_min = x, } } - // 更新最大值，检查时间差并在超过10秒时清零 + // Update maximum value, check time difference and reset if over 10 seconds let mut current_max = self.max_time_bits.load(Ordering::Relaxed); let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed); - // 检查最大值的时间戳是否超过10秒（10_000_000_000纳秒） + // Check if max value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds) let time_diff_nanos = now_nanos.saturating_sub(max_timestamp); if time_diff_nanos > 10_000_000_000 { - // 超过10秒，清零最大值 + // Over 10 seconds, reset max value self.max_time_bits.store(0, Ordering::Relaxed); self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); current_max = 0; } - // 如果当前时间大于最大值，更新最大值和时间戳 + // If current time is greater than max value, update max value and timestamp while time_bits > current_max { match self.max_time_bits.compare_exchange_weak( current_max, @@ -172,7 +188,7 @@ impl AtomicProcessingTimeStats { Ordering::Relaxed, ) { Ok(_) => { - // 成功更新最大值，同时更新时间戳 + // Successfully updated max value, also update timestamp self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed); break; } @@ -180,13 +196,13 @@ impl AtomicProcessingTimeStats { } } - // 更新累计值（将微秒转换为整数避免浮点累加问题） + // Update cumulative values (convert microseconds to integers to avoid floating point accumulation issues) let total_time_us_int = (time_us * event_count as f64) as u64; self.total_time_us.fetch_add(total_time_us_int, Ordering::Relaxed); self.total_events.fetch_add(event_count, Ordering::Relaxed); } - /// 获取统计值（非阻塞） + /// Get statistics (non-blocking) #[inline] fn get_stats(&self) -> ProcessingTimeStats { let min_bits = self.min_time_bits.load(Ordering::Relaxed); @@ -207,7 +223,7 @@ impl AtomicProcessingTimeStats { } } -/// 处理时间统计结果 +/// Processing time statistics result #[derive(Debug, Clone)] pub struct ProcessingTimeStats { pub min_us: f64, @@ -215,7 +231,7 @@ pub struct ProcessingTimeStats { pub avg_us: f64, } -/// 事件指标快照 +/// Event metrics snapshot #[derive(Debug, Clone)] pub struct EventMetricsSnapshot { pub process_count: u64, @@ -223,19 +239,7 @@ pub struct EventMetricsSnapshot { pub events_per_second: f64, } -/// 背压指标快照 -#[derive(Debug, Clone)] -pub struct BackpressureMetricsSnapshot { - pub total_duration_us: u64, - pub success_count: u64, - pub failure_count: u64, - pub min_permits: u64, - pub max_permits: u64, - pub avg_duration_us: f64, - pub success_rate: f64, -} - -/// 兼容性结构 - 完整的性能指标 +/// Compatibility structure - complete performance metrics #[derive(Debug, Clone)] pub struct PerformanceMetrics { pub uptime: std::time::Duration, @@ -243,25 +247,15 @@ pub struct PerformanceMetrics { pub account_metrics: EventMetricsSnapshot, pub block_meta_metrics: EventMetricsSnapshot, pub processing_stats: ProcessingTimeStats, - pub backpressure_metrics: BackpressureMetricsSnapshot, pub dropped_events_count: u64, } impl PerformanceMetrics { - /// 创建默认的性能指标（兼容性方法） + /// Create default performance metrics (compatibility method) pub fn new() -> Self { let default_metrics = EventMetricsSnapshot { process_count: 0, events_processed: 0, events_per_second: 0.0 }; let default_stats = ProcessingTimeStats { min_us: 0.0, max_us: 0.0, avg_us: 0.0 }; - let default_backpressure = BackpressureMetricsSnapshot { - total_duration_us: 0, - success_count: 0, - failure_count: 0, - min_permits: 0, - max_permits: 0, - avg_duration_us: 0.0, - success_rate: 0.0, - }; Self { uptime: std::time::Duration::ZERO, @@ -269,24 +263,17 @@ impl PerformanceMetrics { account_metrics: default_metrics.clone(), block_meta_metrics: default_metrics, processing_stats: default_stats, - backpressure_metrics: default_backpressure, dropped_events_count: 0, } } } -/// 高性能指标系统 +/// High-performance metrics system #[derive(Debug)] pub struct HighPerformanceMetrics { start_nanos: u64, event_metrics: [AtomicEventMetrics; 3], processing_stats: AtomicProcessingTimeStats, - // 背压相关指标 - backpressure_total_duration_us: AtomicU64, - backpressure_success_count: AtomicU64, - backpressure_failure_count: AtomicU64, - backpressure_min_permits: AtomicU64, - backpressure_max_permits: AtomicU64, // 丢弃事件指标 dropped_events_count: AtomicU64, } @@ -305,12 +292,6 @@ impl HighPerformanceMetrics { AtomicEventMetrics::new(now_nanos), ], processing_stats: AtomicProcessingTimeStats::new(), - // 初始化背压相关指标 - backpressure_total_duration_us: AtomicU64::new(0), - backpressure_success_count: AtomicU64::new(0), - backpressure_failure_count: AtomicU64::new(0), - backpressure_min_permits: AtomicU64::new(u64::MAX), // 初始化为最大值，便于后续比较 - backpressure_max_permits: AtomicU64::new(0), // 初始化丢弃事件指标 dropped_events_count: AtomicU64::new(0), } @@ -341,32 +322,6 @@ impl HighPerformanceMetrics { self.processing_stats.get_stats() } - /// 获取背压指标快照 - #[inline] - pub fn get_backpressure_metrics(&self) -> BackpressureMetricsSnapshot { - let total_duration_us = self.backpressure_total_duration_us.load(Ordering::Relaxed); - let success_count = self.backpressure_success_count.load(Ordering::Relaxed); - let failure_count = self.backpressure_failure_count.load(Ordering::Relaxed); - let min_permits = self.backpressure_min_permits.load(Ordering::Relaxed); - let max_permits = self.backpressure_max_permits.load(Ordering::Relaxed); - - let total_count = success_count + failure_count; - let avg_duration_us = - if total_count > 0 { total_duration_us as f64 / total_count as f64 } else { 0.0 }; - let success_rate = - if total_count > 0 { success_count as f64 / total_count as f64 } else { 0.0 }; - - BackpressureMetricsSnapshot { - total_duration_us, - success_count, - failure_count, - min_permits: if min_permits == u64::MAX { 0 } else { min_permits }, - max_permits, - avg_duration_us, - success_rate, - } - } - /// 获取丢弃事件计数 #[inline] pub fn get_dropped_events_count(&self) -> u64 { @@ -509,19 +464,13 @@ impl MetricsManager { /// 记录慢处理操作 #[inline] - pub fn log_slow_processing( - &self, - processing_time_us: f64, - event_count: usize, - signature: Option, - ) { + pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) { if processing_time_us > SLOW_PROCESSING_THRESHOLD_US { - log::warn!( - "{} slow processing: {:.2}us for {} events, signature: {:?}", + log::debug!( + "{} slow processing: {:.2}us for {} events", self.stream_name, processing_time_us, event_count, - signature ); } } @@ -541,11 +490,6 @@ impl MetricsManager { self.metrics.get_processing_stats() } - /// 获取背压指标 - pub fn get_backpressure_metrics(&self) -> BackpressureMetricsSnapshot { - self.metrics.get_backpressure_metrics() - } - /// 获取丢弃事件计数 pub fn get_dropped_events_count(&self) -> u64 { self.metrics.get_dropped_events_count() @@ -556,22 +500,6 @@ impl MetricsManager { println!("\n📊 {} Performance Metrics", self.stream_name); println!(" Run Time: {:?}", self.get_uptime()); - // 打印背压指标表格 - let backpressure = self.get_backpressure_metrics(); - if backpressure.success_count > 0 || backpressure.failure_count > 0 { - println!("\n🚦 Backpressure Metrics"); - println!("┌──────────────────────┬─────────────┐"); - println!("│ Metric │ Value │"); - println!("├──────────────────────┼─────────────┤"); - println!("│ Success Count │ {:11} │", backpressure.success_count); - println!("│ Failure Count │ {:11} │", backpressure.failure_count); - println!("│ Success Rate │ {:11.2} │", backpressure.success_rate * 100.0); - println!("│ Avg Duration (ms) │ {:11.2} │", backpressure.avg_duration_us / 1000.0); - println!("│ Min Permits │ {:11} │", backpressure.min_permits); - println!("│ Max Permits │ {:11} │", backpressure.max_permits); - println!("└──────────────────────┴─────────────┘"); - } - // 打印丢弃事件指标 let dropped_count = self.get_dropped_events_count(); if dropped_count > 0 { @@ -603,7 +531,7 @@ impl MetricsManager { println!("│ Metric │ Value (us) │"); println!("├───────────────────────┼─────────────┤"); println!("│ Average │ {:9.2} │", stats.avg_us); - println!("│ Minimum │ {:9.2} │", stats.min_us); + println!("│ Minimum within 10s │ {:9.2} │", stats.min_us); println!("│ Maximum within 10s │ {:9.2} │", stats.max_us); println!("└───────────────────────┴─────────────┘"); @@ -648,7 +576,6 @@ impl MetricsManager { account_metrics: self.get_event_metrics(EventType::Account), block_meta_metrics: self.get_event_metrics(EventType::BlockMeta), processing_stats: self.get_processing_stats(), - backpressure_metrics: self.metrics.get_backpressure_metrics(), dropped_events_count: self.metrics.get_dropped_events_count(), } } @@ -678,76 +605,9 @@ impl MetricsManager { event_type: MetricsEventType, events_processed: u64, processing_time_us: f64, - signature: Option, ) { self.record_events(event_type, events_processed, processing_time_us); - self.log_slow_processing(processing_time_us, events_processed as usize, signature); - } - - /// 记录背压相关的metrics - #[inline] - pub fn record_backpressure_metrics( - &self, - backpressure_duration: std::time::Duration, - success: bool, - available_permits: usize, - ) { - if !self.enable_metrics { - return; - } - - let duration_us = backpressure_duration.as_micros() as u64; - let permits = available_permits as u64; - - // 记录总持续时间 - self.metrics.backpressure_total_duration_us.fetch_add(duration_us, Ordering::Relaxed); - - // 记录成功/失败计数 - if success { - self.metrics.backpressure_success_count.fetch_add(1, Ordering::Relaxed); - } else { - self.metrics.backpressure_failure_count.fetch_add(1, Ordering::Relaxed); - } - - // 更新最小许可数（使用 compare_exchange_weak 循环） - let mut current_min = self.metrics.backpressure_min_permits.load(Ordering::Relaxed); - while permits < current_min { - match self.metrics.backpressure_min_permits.compare_exchange_weak( - current_min, - permits, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(x) => current_min = x, - } - } - - // 更新最大许可数 - let mut current_max = self.metrics.backpressure_max_permits.load(Ordering::Relaxed); - while permits > current_max { - match self.metrics.backpressure_max_permits.compare_exchange_weak( - current_max, - permits, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(x) => current_max = x, - } - } - - // 记录慢背压操作的日志 - if duration_us > 10_000 { - // 超过10ms的背压认为是慢操作 - log::warn!( - "{} slow backpressure: {:.2}ms, success: {}, available_permits: {}", - self.stream_name, - duration_us as f64 / 1000.0, - success, - available_permits - ); - } + self.log_slow_processing(processing_time_us, events_processed as usize); } /// 增加丢弃事件计数 @@ -762,7 +622,7 @@ impl MetricsManager { // 每丢弃1000个事件记录一次警告日志 if new_count % 1000 == 0 { - log::warn!("{} dropped events count reached: {}", self.stream_name, new_count); + log::debug!("{} dropped events count reached: {}", self.stream_name, new_count); } } @@ -774,17 +634,22 @@ impl MetricsManager { } // 原子地增加丢弃事件计数 - let new_count = self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count; + let new_count = + self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count; // 记录批量丢弃事件的日志 if count > 1 { - log::warn!("{} dropped batch of {} events, total dropped: {}", - self.stream_name, count, new_count); + log::debug!( + "{} dropped batch of {} events, total dropped: {}", + self.stream_name, + count, + new_count + ); } // 每丢弃1000个事件记录一次警告日志 if new_count % 1000 == 0 || (new_count / 1000) != ((new_count - count) / 1000) { - log::warn!("{} dropped events count reached: {}", self.stream_name, new_count); + log::debug!("{} dropped events count reached: {}", self.stream_name, new_count); } } } diff --git a/src/streaming/common/mod.rs b/src/streaming/common/mod.rs index 011bab8..55a76aa 100644 --- a/src/streaming/common/mod.rs +++ b/src/streaming/common/mod.rs @@ -1,15 +1,15 @@ // 公用模块 - 包含流处理相关的通用功能 pub mod config; pub mod metrics; -pub mod batch; pub mod constants; pub mod subscription; pub mod event_processor; +pub mod simd_utils; // 重新导出主要类型 pub use config::*; pub use metrics::*; -pub use batch::*; pub use constants::*; pub use subscription::*; -pub use event_processor::*; \ No newline at end of file +pub use event_processor::*; +pub use simd_utils::*; \ No newline at end of file diff --git a/src/streaming/common/simd_utils.rs b/src/streaming/common/simd_utils.rs new file mode 100644 index 0000000..8aaf1f0 --- /dev/null +++ b/src/streaming/common/simd_utils.rs @@ -0,0 +1,295 @@ +use wide::*; + +/// SIMD-accelerated data parsing utilities +pub struct SimdUtils; + +impl SimdUtils { + /// SIMD-accelerated byte array comparison + /// For arrays with length >= 16, uses SIMD instructions for fast comparison + #[inline(always)] + pub fn fast_bytes_equal(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + + let len = a.len(); + + // For small arrays, use standard comparison directly + if len < 16 { + return a == b; + } + + // Use SIMD to process 16-byte chunks + let chunks = len / 16; + let remainder = len % 16; + + // Process complete 16-byte chunks + for i in 0..chunks { + let offset = i * 16; + let chunk_a = u8x16::from(&a[offset..offset + 16]); + let chunk_b = u8x16::from(&b[offset..offset + 16]); + + if !chunk_a.cmp_eq(chunk_b).all() { + return false; + } + } + + // Process remaining bytes + if remainder > 0 { + let start = chunks * 16; + return &a[start..] == &b[start..]; + } + + true + } + + /// Fast discriminator matching, specifically for instruction discriminator comparison + #[inline(always)] + pub fn fast_discriminator_match(data: &[u8], discriminator: &[u8]) -> bool { + if data.len() < discriminator.len() { + return false; + } + + let disc_len = discriminator.len(); + + // Optimize for common discriminator lengths + match disc_len { + 1 => data[0] == discriminator[0], + 2 => { + let data_u16 = u16::from_le_bytes([data[0], data[1]]); + let disc_u16 = u16::from_le_bytes([discriminator[0], discriminator[1]]); + data_u16 == disc_u16 + } + 4 => { + let data_u32 = u32::from_le_bytes([data[0], data[1], data[2], data[3]]); + let disc_u32 = u32::from_le_bytes([ + discriminator[0], + discriminator[1], + discriminator[2], + discriminator[3], + ]); + data_u32 == disc_u32 + } + 8 => { + let data_u64 = u64::from_le_bytes([ + data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], + ]); + let disc_u64 = u64::from_le_bytes([ + discriminator[0], + discriminator[1], + discriminator[2], + discriminator[3], + discriminator[4], + discriminator[5], + discriminator[6], + discriminator[7], + ]); + data_u64 == disc_u64 + } + 16 => { + // Use SIMD to process 16-byte discriminators + let data_chunk = u8x16::from(&data[..16]); + let disc_chunk = u8x16::from(discriminator); + data_chunk.cmp_eq(disc_chunk).all() + } + _ => { + // For other lengths, use generic SIMD comparison + Self::fast_bytes_equal(&data[..disc_len], discriminator) + } + } + } + + /// SIMD-accelerated memory search to find specific patterns in data + #[inline(always)] + pub fn find_pattern_simd(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || haystack.len() < needle.len() { + return None; + } + + let needle_len = needle.len(); + let haystack_len = haystack.len(); + + // For single-byte search, use optimized method + if needle_len == 1 { + let target = needle[0]; + return haystack.iter().position(|&b| b == target); + } + + // For multi-byte search, use SIMD acceleration + if needle_len <= 16 && haystack_len >= 16 { + let first_byte = needle[0]; + let chunks = (haystack_len - needle_len + 1) / 16; + + for chunk_idx in 0..chunks { + let start = chunk_idx * 16; + let end = std::cmp::min(start + 16, haystack_len - needle_len + 1); + + // Use SIMD to find first byte matches + let chunk = &haystack[start..start + 16]; + let target_vec = u8x16::splat(first_byte); + let chunk_vec = u8x16::from(chunk); + let matches = chunk_vec.cmp_eq(target_vec); + + // Check each match position + let matches_array: [u8; 16] = matches.into(); + for i in 0..16 { + if start + i >= end { + break; + } + + if matches_array[i] != 0 && start + i + needle_len <= haystack_len { + if Self::fast_bytes_equal( + &haystack[start + i..start + i + needle_len], + needle, + ) { + return Some(start + i); + } + } + } + } + + // Process remaining part + let remaining_start = chunks * 16; + for i in remaining_start..=(haystack_len - needle_len) { + if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) { + return Some(i); + } + } + } else { + // Fallback to standard search + for i in 0..=(haystack_len - needle_len) { + if Self::fast_bytes_equal(&haystack[i..i + needle_len], needle) { + return Some(i); + } + } + } + + None + } + + /// SIMD-accelerated data validation to check if data conforms to specific format + #[inline(always)] + pub fn validate_data_format(data: &[u8], min_length: usize) -> bool { + if data.len() < min_length { + return false; + } + + true + } + + /// Fast checksum calculation (maintains API consistency) + #[inline(always)] + pub fn fast_checksum(data: &[u8]) -> u32 { + // Simplified implementation, directly sum all bytes + data.iter().map(|&b| b as u32).sum() + } + + /// SIMD-accelerated data copy (for large data blocks) + #[inline(always)] + pub fn fast_copy(src: &[u8], dst: &mut [u8]) { + if src.len() != dst.len() { + panic!("Source and destination must have the same length"); + } + + let len = src.len(); + + if len >= 32 { + // Use 32-byte SIMD copy + let chunks = len / 32; + + for i in 0..chunks { + let start = i * 32; + let src_chunk1 = u8x16::from(&src[start..start + 16]); + let src_chunk2 = u8x16::from(&src[start + 16..start + 32]); + + let chunk1_array: [u8; 16] = src_chunk1.into(); + let chunk2_array: [u8; 16] = src_chunk2.into(); + + dst[start..start + 16].copy_from_slice(&chunk1_array); + dst[start + 16..start + 32].copy_from_slice(&chunk2_array); + } + + // Process remaining bytes + let remaining_start = chunks * 32; + dst[remaining_start..].copy_from_slice(&src[remaining_start..]); + } else { + // For small data, use standard copy + dst.copy_from_slice(src); + } + } + + /// SIMD-accelerated account indices validation + /// Validates that all indices in the account index array are less than the total account count + #[inline(always)] + pub fn validate_account_indices_simd(indices: &[u8], account_count: usize) -> bool { + if indices.is_empty() { + return true; + } + + let max_valid_index = account_count as u8; + + // For small arrays, use standard comparison directly + if indices.len() < 16 { + return indices.iter().all(|&idx| idx < max_valid_index); + } + + // Use SIMD for batch loading and comparison + let chunks = indices.len() / 16; + let remainder = indices.len() % 16; + + // Process complete 16-byte chunks + for i in 0..chunks { + let start = i * 16; + let indices_chunk = u8x16::from(&indices[start..start + 16]); + + // Convert SIMD vector to array for fast batch checking + let indices_array: [u8; 16] = indices_chunk.into(); + + // Use unrolled loop for fast comparison, compiler will optimize this + if indices_array[0] >= max_valid_index + || indices_array[1] >= max_valid_index + || indices_array[2] >= max_valid_index + || indices_array[3] >= max_valid_index + || indices_array[4] >= max_valid_index + || indices_array[5] >= max_valid_index + || indices_array[6] >= max_valid_index + || indices_array[7] >= max_valid_index + || indices_array[8] >= max_valid_index + || indices_array[9] >= max_valid_index + || indices_array[10] >= max_valid_index + || indices_array[11] >= max_valid_index + || indices_array[12] >= max_valid_index + || indices_array[13] >= max_valid_index + || indices_array[14] >= max_valid_index + || indices_array[15] >= max_valid_index + { + return false; + } + } + + // Process remaining bytes + if remainder > 0 { + let remaining_start = chunks * 16; + return indices[remaining_start..].iter().all(|&idx| idx < max_valid_index); + } + + true + } + + /// SIMD-accelerated instruction data validation + /// Validates basic format and length requirements of instruction data + #[inline(always)] + pub fn validate_instruction_data_simd( + data: &[u8], + min_length: usize, + discriminator_length: usize, + ) -> bool { + // Basic length check + if data.len() < min_length || data.len() < discriminator_length { + return false; + } + + // Use existing data format validation + Self::validate_data_format(data, min_length) + } +} diff --git a/src/streaming/event_parser/common/mod.rs b/src/streaming/event_parser/common/mod.rs index c74b304..3e9d088 100755 --- a/src/streaming/event_parser/common/mod.rs +++ b/src/streaming/event_parser/common/mod.rs @@ -2,22 +2,12 @@ pub mod types; pub mod utils; pub mod filter; -pub const EMPTY_ID: &str = ""; - /// 自动生成UnifiedEvent trait实现的宏 #[macro_export] macro_rules! impl_unified_event { // 带有自定义ID表达式的版本 ($struct_name:ident, $($field:ident),*) => { impl $crate::streaming::event_parser::core::traits::UnifiedEvent for $struct_name { - fn id(&self) -> &str { - &self.metadata.id - } - - fn clear_id(&mut self) { - self.metadata.id = $crate::streaming::event_parser::common::EMPTY_ID.to_string(); - } - fn event_type(&self) -> $crate::streaming::event_parser::common::types::EventType { self.metadata.event_type.clone() } @@ -66,6 +56,10 @@ macro_rules! impl_unified_event { self.metadata.set_swap_data(swap_data); } + fn swap_data_is_parsed(&self) -> bool { + self.metadata.swap_data.is_some() + } + fn instruction_outer_index(&self) -> i64 { self.metadata.instruction_outer_index } diff --git a/src/streaming/event_parser/common/types.rs b/src/streaming/event_parser/common/types.rs index 60c1e1a..b086d8f 100755 --- a/src/streaming/event_parser/common/types.rs +++ b/src/streaming/event_parser/common/types.rs @@ -2,26 +2,23 @@ use borsh::{BorshDeserialize, BorshSerialize}; use crossbeam_queue::ArrayQueue; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; -use solana_transaction_status::{InnerInstruction, UiInstruction}; -use std::{ - fmt, - hash::{DefaultHasher, Hash, Hasher}, - str::FromStr, - sync::Arc, -}; +use std::{borrow::Cow, fmt, str::FromStr, sync::Arc}; use crate::{ match_event, - streaming::event_parser::{ - protocols::{ - bonk::BonkTradeEvent, - pumpfun::PumpFunTradeEvent, - pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}, - raydium_amm_v4::RaydiumAmmV4SwapEvent, - raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event}, - raydium_cpmm::RaydiumCpmmSwapEvent, + streaming::{ + common::SimdUtils, + event_parser::{ + protocols::{ + bonk::BonkTradeEvent, + pumpfun::PumpFunTradeEvent, + pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}, + raydium_amm_v4::RaydiumAmmV4SwapEvent, + raydium_clmm::{RaydiumClmmSwapEvent, RaydiumClmmSwapV2Event}, + raydium_cpmm::RaydiumCpmmSwapEvent, + }, + UnifiedEvent, }, - UnifiedEvent, }, }; @@ -293,8 +290,7 @@ pub struct SwapData { Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, )] pub struct EventMetadata { - pub id: String, - pub signature: String, + pub signature: Cow<'static, str>, pub slot: u64, pub transaction_index: Option, // 新增：交易在slot中的索引 pub block_time: i64, @@ -312,8 +308,7 @@ pub struct EventMetadata { impl EventMetadata { #[allow(clippy::too_many_arguments)] pub fn new( - id: String, - signature: String, + signature: Cow<'static, str>, slot: u64, block_time: i64, block_time_ms: i64, @@ -326,7 +321,6 @@ impl EventMetadata { transaction_index: Option, ) -> Self { Self { - id, signature, slot, block_time, @@ -343,10 +337,6 @@ impl EventMetadata { } } - pub fn set_id(&mut self, id: String) { - self.id = format!("{}-{}-{}", self.signature, self.event_type, id); - } - pub fn set_swap_data(&mut self, swap_data: SwapData) { self.swap_data = Some(swap_data); } @@ -463,6 +453,12 @@ pub fn parse_swap_data_from_next_instructions( break; } let data = &compiled.data; + + // 使用 SIMD 验证数据格式 + if !SimdUtils::validate_data_format(data, 8) { + continue; + } + let get_pubkey = |i: usize| accounts[compiled.accounts[i] as usize]; let (source, destination, amount) = match data[0] { 12 if compiled.accounts.len() >= 4 => { diff --git a/src/streaming/event_parser/core/account_event_parser.rs b/src/streaming/event_parser/core/account_event_parser.rs index 030266a..0efa7be 100644 --- a/src/streaming/event_parser/core/account_event_parser.rs +++ b/src/streaming/event_parser/core/account_event_parser.rs @@ -1,11 +1,13 @@ +use std::borrow::Cow; use std::collections::HashMap; use std::sync::OnceLock; use solana_sdk::pubkey::Pubkey; +use crate::streaming::common::SimdUtils; use crate::streaming::event_parser::common::filter::EventTypeFilter; use crate::streaming::event_parser::common::{EventMetadata, EventType, ProtocolType}; -use crate::streaming::event_parser::core::traits::UnifiedEvent; +use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock}; use crate::streaming::event_parser::protocols::bonk::parser::BONK_PROGRAM_ID; use crate::streaming::event_parser::protocols::pumpfun::parser::PUMPFUN_PROGRAM_ID; use crate::streaming::event_parser::protocols::pumpswap::parser::PUMPSWAP_PROGRAM_ID; @@ -35,7 +37,10 @@ static PROTOCOL_CONFIGS_CACHE: OnceLock, event_type_filter: Option) -> Vec { + pub fn configs( + protocols: &[Protocol], + event_type_filter: Option<&EventTypeFilter>, + ) -> Vec { let protocols_map = PROTOCOL_CONFIGS_CACHE.get_or_init(|| { let mut map: HashMap> = HashMap::new(); map.insert(Protocol::PumpSwap, vec![ @@ -145,32 +150,39 @@ impl AccountEventParser { }); let mut configs = vec![]; + let empty_vec = vec![]; for protocol in protocols { - let protocol_configs = protocols_map.get(&protocol).unwrap_or(&vec![]).clone(); - let filtered_configs: Vec = protocol_configs.into_iter().filter(|config| { - event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true) - }).collect(); + let protocol_configs = protocols_map.get(protocol).unwrap_or(&empty_vec); + let filtered_configs: Vec = protocol_configs + .iter() + .filter(|config| { + event_type_filter + .map(|filter| filter.include.contains(&config.event_type)) + .unwrap_or(true) + }) + .cloned() + .collect(); configs.extend(filtered_configs); } configs } pub fn parse_account_event( - protocols: Vec, + protocols: &[Protocol], account: AccountPretty, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> Option> { let configs = Self::configs(protocols, event_type_filter); for config in configs { if account.owner == config.program_id - && account.data[..config.account_discriminator.len()] - == *config.account_discriminator + && SimdUtils::fast_discriminator_match(&account.data, config.account_discriminator) { + let signature_str = Cow::Owned(account.signature.to_string()); let event = (config.account_parser)( &account, EventMetadata { slot: account.slot, - signature: account.signature.to_string(), + signature: signature_str, protocol: config.protocol_type, event_type: config.event_type, program_id: config.program_id, @@ -180,7 +192,7 @@ impl AccountEventParser { ); if let Some(mut event) = event { event.set_program_handle_time_consuming_us( - chrono::Utc::now().timestamp_micros() - account.program_received_time_us, + get_high_perf_clock().elapsed_micros_since(account.program_received_time_us), ); return Some(event); } diff --git a/src/streaming/event_parser/core/common_event_parser.rs b/src/streaming/event_parser/core/common_event_parser.rs index 6ec37ae..ef44d96 100644 --- a/src/streaming/event_parser/core/common_event_parser.rs +++ b/src/streaming/event_parser/core/common_event_parser.rs @@ -1,4 +1,4 @@ -use crate::streaming::event_parser::core::traits::UnifiedEvent; +use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock}; use crate::streaming::event_parser::protocols::block::block_meta_event::BlockMetaEvent; pub struct CommonEventParser {} @@ -17,7 +17,7 @@ impl CommonEventParser { program_received_time_us, ); block_meta_event.set_program_handle_time_consuming_us( - chrono::Utc::now().timestamp_micros() - program_received_time_us, + get_high_perf_clock().elapsed_micros_since(program_received_time_us), ); Box::new(block_meta_event) } diff --git a/src/streaming/event_parser/core/global_state.rs b/src/streaming/event_parser/core/global_state.rs index 5e75c0f..c14142e 100644 --- a/src/streaming/event_parser/core/global_state.rs +++ b/src/streaming/event_parser/core/global_state.rs @@ -1,92 +1,174 @@ use solana_sdk::pubkey::Pubkey; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use dashmap::DashMap; +use std::collections::BTreeSet; -/// Global state management, thread-safe implementation without locks +const MAX_SLOTS: usize = 1000; +const CLEANUP_BATCH_SIZE: usize = 100; + +/// Slot-based trader addresses, completely lock-free +#[derive(Default)] +struct SlotAddresses { + /// Developer addresses for this slot + dev_addresses: BTreeSet, + /// Bonk developer addresses for this slot + bonk_dev_addresses: BTreeSet, +} + +/// High-performance global state with lock-free slot-based storage pub struct GlobalState { - /// Last processed slot - last_slot: AtomicU64, - /// Developer address array - dev_addresses: parking_lot::RwLock>, - /// Bonk developer address array - bonk_dev_addresses: parking_lot::RwLock>, + /// Slot -> trader addresses mapping (lock-free concurrent hashmap) + slot_data: DashMap, + /// Current slot count for capacity management + slot_count: AtomicUsize, + /// Generation counter to handle cleanup races + generation: AtomicU64, } impl GlobalState { - /// Create a new global state instance + /// Create a new high-performance global state instance pub fn new() -> Self { Self { - last_slot: AtomicU64::new(0), - dev_addresses: parking_lot::RwLock::new(Vec::new()), - bonk_dev_addresses: parking_lot::RwLock::new(Vec::new()), + slot_data: DashMap::new(), + slot_count: AtomicUsize::new(0), + generation: AtomicU64::new(0), } } - /// Get current slot - pub fn get_last_slot(&self) -> u64 { - self.last_slot.load(Ordering::Relaxed) - } + /// Lock-free capacity management - cleanup old slots when limit exceeded + fn maybe_cleanup(&self) { + let current_count = self.slot_count.load(Ordering::Relaxed); + if current_count <= MAX_SLOTS { + return; + } - /// Update slot, clear arrays if slot changes - pub fn update_slot(&self, new_slot: u64) { - let old_slot = self.last_slot.swap(new_slot, Ordering::Relaxed); + // Use CAS to ensure only one thread performs cleanup + let gen = self.generation.load(Ordering::Relaxed); + if self.generation.compare_exchange_weak(gen, gen + 1, Ordering::Acquire, Ordering::Relaxed).is_err() { + return; // Another thread is cleaning up + } - if old_slot != new_slot { - // Clear arrays when slot changes - let mut dev_addresses = self.dev_addresses.write(); - let mut bonk_dev_addresses = self.bonk_dev_addresses.write(); + // Collect oldest slots (BTreeMap naturally orders by key) + let mut slots_to_remove: Vec = self.slot_data.iter() + .map(|entry| *entry.key()) + .collect(); + + if slots_to_remove.len() <= MAX_SLOTS { + return; // Race condition, already cleaned up + } + + slots_to_remove.sort_unstable(); + slots_to_remove.truncate(CLEANUP_BATCH_SIZE); - dev_addresses.clear(); - bonk_dev_addresses.clear(); + // Remove old slots atomically + for slot in slots_to_remove { + self.slot_data.remove(&slot); + self.slot_count.fetch_sub(1, Ordering::Relaxed); } } - /// Add developer address - pub fn add_dev_address(&self, address: Pubkey) { - let mut dev_addresses = self.dev_addresses.write(); - if !dev_addresses.contains(&address) { - dev_addresses.push(address); - } + /// Add developer address for a specific slot (lock-free) + pub fn add_dev_address(&self, slot: u64, address: Pubkey) { + self.maybe_cleanup(); + + self.slot_data.entry(slot) + .and_modify(|addresses| { + addresses.dev_addresses.insert(address); + }) + .or_insert_with(|| { + self.slot_count.fetch_add(1, Ordering::Relaxed); + let mut slot_addr = SlotAddresses::default(); + slot_addr.dev_addresses.insert(address); + slot_addr + }); } - /// Check if address is a developer address + /// Add Bonk developer address for a specific slot (lock-free) + pub fn add_bonk_dev_address(&self, slot: u64, address: Pubkey) { + self.maybe_cleanup(); + + self.slot_data.entry(slot) + .and_modify(|addresses| { + addresses.bonk_dev_addresses.insert(address); + }) + .or_insert_with(|| { + self.slot_count.fetch_add(1, Ordering::Relaxed); + let mut slot_addr = SlotAddresses::default(); + slot_addr.bonk_dev_addresses.insert(address); + slot_addr + }); + } + + /// High-performance: Check if address is a developer address in specific slot (O(log m)) + pub fn is_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool { + self.slot_data.get(&slot) + .map(|entry| entry.dev_addresses.contains(address)) + .unwrap_or(false) + } + + /// High-performance: Check if address is a Bonk developer address in specific slot (O(log m)) + pub fn is_bonk_dev_address_in_slot(&self, slot: u64, address: &Pubkey) -> bool { + self.slot_data.get(&slot) + .map(|entry| entry.bonk_dev_addresses.contains(address)) + .unwrap_or(false) + } + + /// Check if address is a developer address in any slot (lock-free scan, slower) pub fn is_dev_address(&self, address: &Pubkey) -> bool { - let dev_addresses = self.dev_addresses.read(); - dev_addresses.contains(address) + self.slot_data.iter().any(|entry| entry.dev_addresses.contains(address)) } - /// Add Bonk developer address - pub fn add_bonk_dev_address(&self, address: Pubkey) { - let mut bonk_dev_addresses = self.bonk_dev_addresses.write(); - if !bonk_dev_addresses.contains(&address) { - bonk_dev_addresses.push(address); - } - } - - /// Check if address is a Bonk developer address + /// Check if address is a Bonk developer address in any slot (lock-free scan, slower) pub fn is_bonk_dev_address(&self, address: &Pubkey) -> bool { - let bonk_dev_addresses = self.bonk_dev_addresses.read(); - bonk_dev_addresses.contains(address) + self.slot_data.iter().any(|entry| entry.bonk_dev_addresses.contains(address)) } - /// Get all developer addresses + /// Get all developer addresses from all slots (lock-free aggregation) pub fn get_dev_addresses(&self) -> Vec { - let dev_addresses = self.dev_addresses.read(); - dev_addresses.clone() + let mut all_addresses = BTreeSet::new(); + for entry in self.slot_data.iter() { + for addr in &entry.dev_addresses { + all_addresses.insert(*addr); + } + } + all_addresses.into_iter().collect() } - /// Get all Bonk developer addresses + /// Get all Bonk developer addresses from all slots (lock-free aggregation) pub fn get_bonk_dev_addresses(&self) -> Vec { - let bonk_dev_addresses = self.bonk_dev_addresses.read(); - bonk_dev_addresses.clone() + let mut all_addresses = BTreeSet::new(); + for entry in self.slot_data.iter() { + for addr in &entry.bonk_dev_addresses { + all_addresses.insert(*addr); + } + } + all_addresses.into_iter().collect() } - /// Clear all data - pub fn clear_all_data(&self) { - let mut dev_addresses = self.dev_addresses.write(); - let mut bonk_dev_addresses = self.bonk_dev_addresses.write(); + /// Get developer addresses for a specific slot + pub fn get_dev_addresses_for_slot(&self, slot: u64) -> Vec { + self.slot_data.get(&slot) + .map(|entry| entry.dev_addresses.iter().copied().collect()) + .unwrap_or_default() + } - dev_addresses.clear(); - bonk_dev_addresses.clear(); + /// Get Bonk developer addresses for a specific slot + pub fn get_bonk_dev_addresses_for_slot(&self, slot: u64) -> Vec { + self.slot_data.get(&slot) + .map(|entry| entry.bonk_dev_addresses.iter().copied().collect()) + .unwrap_or_default() + } + + /// Get current slot count + pub fn get_slot_count(&self) -> usize { + self.slot_count.load(Ordering::Relaxed) + } + + /// Clear all data (lock-free) + pub fn clear_all_data(&self) { + self.slot_data.clear(); + self.slot_count.store(0, Ordering::Relaxed); + self.generation.store(0, Ordering::Relaxed); } } @@ -105,14 +187,9 @@ pub fn get_global_state() -> &'static GlobalState { &GLOBAL_STATE } -/// Convenience function: Update slot -pub fn update_slot(slot: u64) { - get_global_state().update_slot(slot); -} - -/// Convenience function: Add developer address -pub fn add_dev_address(address: Pubkey) { - get_global_state().add_dev_address(address); +/// Convenience function: Add developer address for a specific slot +pub fn add_dev_address(slot: u64, address: Pubkey) { + get_global_state().add_dev_address(slot, address); } /// Convenience function: Check if address is a developer address @@ -120,9 +197,9 @@ pub fn is_dev_address(address: &Pubkey) -> bool { get_global_state().is_dev_address(address) } -/// Convenience function: Add Bonk developer address -pub fn add_bonk_dev_address(address: Pubkey) { - get_global_state().add_bonk_dev_address(address); +/// Convenience function: Add Bonk developer address for a specific slot +pub fn add_bonk_dev_address(slot: u64, address: Pubkey) { + get_global_state().add_bonk_dev_address(slot, address); } /// Convenience function: Check if address is a Bonk developer address @@ -139,3 +216,28 @@ pub fn get_dev_addresses() -> Vec { pub fn get_bonk_dev_addresses() -> Vec { get_global_state().get_bonk_dev_addresses() } + +/// Convenience function: Get developer addresses for a specific slot +pub fn get_dev_addresses_for_slot(slot: u64) -> Vec { + get_global_state().get_dev_addresses_for_slot(slot) +} + +/// Convenience function: Get Bonk developer addresses for a specific slot +pub fn get_bonk_dev_addresses_for_slot(slot: u64) -> Vec { + get_global_state().get_bonk_dev_addresses_for_slot(slot) +} + +/// Convenience function: Get current slot count +pub fn get_slot_count() -> usize { + get_global_state().get_slot_count() +} + +/// High-performance: Check if address is a developer address in specific slot +pub fn is_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool { + get_global_state().is_dev_address_in_slot(slot, address) +} + +/// High-performance: Check if address is a Bonk developer address in specific slot +pub fn is_bonk_dev_address_in_slot(slot: u64, address: &Pubkey) -> bool { + get_global_state().is_bonk_dev_address_in_slot(slot, address) +} diff --git a/src/streaming/event_parser/core/traits.rs b/src/streaming/event_parser/core/traits.rs index db74152..65c93de 100755 --- a/src/streaming/event_parser/core/traits.rs +++ b/src/streaming/event_parser/core/traits.rs @@ -8,33 +8,141 @@ use solana_transaction_status::{ EncodedConfirmedTransactionWithStatusMeta, InnerInstruction, InnerInstructions, TransactionWithStatusMeta, UiInstruction, }; +use std::borrow::Cow; use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; +use std::time::Instant; -use super::global_state::{add_dev_address, is_dev_address, update_slot}; - -use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData}; -use crate::streaming::event_parser::core::global_state::{ - add_bonk_dev_address, is_bonk_dev_address, +use super::global_state::{ + add_bonk_dev_address, add_dev_address, is_bonk_dev_address, is_dev_address, }; + +use crate::streaming::common::simd_utils::SimdUtils; +use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData}; use crate::streaming::event_parser::protocols::pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent}; use crate::streaming::event_parser::{ - common::{utils::*, EventMetadata, EventType, ProtocolType}, + common::{EventMetadata, EventType, ProtocolType}, protocols::{ bonk::{BonkPoolCreateEvent, BonkTradeEvent}, pumpfun::{PumpFunCreateTokenEvent, PumpFunTradeEvent}, }, }; +/// 高性能时钟管理器，减少系统调用开销 +#[derive(Debug)] +pub struct HighPerformanceClock { + /// 基准时间点（程序启动时的单调时钟时间） + base_instant: Instant, + /// 基准时间点对应的UTC时间戳（微秒） + base_timestamp_us: i64, +} + +impl HighPerformanceClock { + /// 创建新的高性能时钟 + pub fn new() -> Self { + let base_instant = Instant::now(); + let base_timestamp_us = chrono::Utc::now().timestamp_micros(); + + Self { base_instant, base_timestamp_us } + } + + /// 获取当前时间戳（微秒），使用单调时钟计算，避免系统调用 + #[inline(always)] + pub fn now_micros(&self) -> i64 { + let elapsed = self.base_instant.elapsed(); + self.base_timestamp_us + elapsed.as_micros() as i64 + } + + /// 计算从指定时间戳到现在的消耗时间（微秒） + #[inline(always)] + pub fn elapsed_micros_since(&self, start_timestamp_us: i64) -> i64 { + self.now_micros() - start_timestamp_us + } +} + +impl Default for HighPerformanceClock { + fn default() -> Self { + Self::new() + } +} + +/// 全局高性能时钟实例（使用OnceCell避免重复初始化） +static HIGH_PERF_CLOCK: once_cell::sync::OnceCell = + once_cell::sync::OnceCell::new(); + +/// 获取全局高性能时钟实例 +#[inline(always)] +pub fn get_high_perf_clock() -> &'static HighPerformanceClock { + HIGH_PERF_CLOCK.get_or_init(HighPerformanceClock::new) +} + +/// 轻量级事件包装器，避免频繁的Box分配 +#[derive(Debug)] +pub struct EventWrapper { + pub event: T, +} + +impl EventWrapper { + #[inline] + pub fn new(event: T) -> Self { + Self { event } + } + + #[inline] + pub fn into_boxed(self) -> Box { + Box::new(self.event) + } +} + +/// 高性能账户公钥缓存，避免重复Vec分配 +#[derive(Debug)] +pub struct AccountPubkeyCache { + /// 预分配的账户公钥向量，避免每次重新分配 + cache: Vec, +} + +impl AccountPubkeyCache { + /// 创建新的账户公钥缓存 + pub fn new() -> Self { + Self { + cache: Vec::with_capacity(32), // 预分配32个位置，覆盖大多数交易 + } + } + + /// 从指令账户索引构建账户公钥向量，重用缓存内存 + #[inline] + pub fn build_account_pubkeys( + &mut self, + instruction_accounts: &[u8], + all_accounts: &[Pubkey], + ) -> &[Pubkey] { + self.cache.clear(); + + // 确保容量足够，避免动态扩容 + if self.cache.capacity() < instruction_accounts.len() { + self.cache.reserve(instruction_accounts.len() - self.cache.capacity()); + } + + // 快速填充账户公钥 + for &idx in instruction_accounts.iter() { + if (idx as usize) < all_accounts.len() { + self.cache.push(all_accounts[idx as usize]); + } + } + + &self.cache + } +} + +impl Default for AccountPubkeyCache { + fn default() -> Self { + Self::new() + } +} + /// Unified Event Interface - All protocol events must implement this trait pub trait UnifiedEvent: Debug + Send + Sync { - /// Get event ID - fn id(&self) -> &str; - - /// Set event ID - fn clear_id(&mut self); - /// Get event type fn event_type(&self) -> EventType; @@ -70,6 +178,9 @@ pub trait UnifiedEvent: Debug + Send + Sync { /// Set swap data fn set_swap_data(&mut self, swap_data: SwapData); + /// swap_data is parsed + fn swap_data_is_parsed(&self) -> bool; + /// Get index fn instruction_outer_index(&self) -> i64; fn instruction_inner_index(&self) -> Option; @@ -343,7 +454,6 @@ pub trait EventParser: Send + Sync { let versioned_tx = match transaction.transaction.transaction.decode() { Some(tx) => tx, None => { - println!("Failed to decode transaction"); return Ok(()); } }; @@ -363,6 +473,7 @@ pub trait EventParser: Send + Sync { if let UiInstruction::Compiled(ui_compiled) = ui_instruction { // 解码base58编码的data if let Ok(data) = bs58::decode(&ui_compiled.data).into_vec() { + // base64解码 let compiled_instruction = CompiledInstruction { program_id_index: ui_compiled.program_id_index, accounts: ui_compiled.accounts.clone(), @@ -565,6 +676,8 @@ pub struct GenericEventParser { pub program_ids: Vec, // pub inner_instruction_configs: HashMap, Vec>, pub instruction_configs: HashMap, Vec>, + /// 账户公钥缓存，避免重复分配 + pub account_cache: parking_lot::Mutex, } impl GenericEventParser { @@ -580,7 +693,10 @@ impl GenericEventParser { .push(config.clone()); } - Self { program_ids, instruction_configs } + // 初始化账户缓存 + let account_cache = parking_lot::Mutex::new(AccountPubkeyCache::new()); + + Self { program_ids, instruction_configs, account_cache } } /// 通用的内联指令解析方法 @@ -598,11 +714,11 @@ impl GenericEventParser { transaction_index: Option, ) -> Option> { if let Some(parser) = config.inner_instruction_parser { + let signature_str = Cow::Owned(signature.to_string()); let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); let block_time_ms = timestamp.seconds * 1000 + (timestamp.nanos as i64) / 1_000_000; let metadata = EventMetadata::new( - signature.to_string(), - signature.to_string(), + signature_str, slot, timestamp.seconds, block_time_ms, @@ -636,11 +752,11 @@ impl GenericEventParser { transaction_index: Option, ) -> Option> { if let Some(parser) = config.instruction_parser { + let signature_str = Cow::Owned(signature.to_string()); let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 }); let block_time_ms = timestamp.seconds * 1000 + (timestamp.nanos as i64) / 1_000_000; let metadata = EventMetadata::new( - signature.to_string(), - signature.to_string(), + signature_str, slot, timestamp.seconds, block_time_ms, @@ -678,7 +794,8 @@ impl EventParser for GenericEventParser { transaction_index: Option, config: &GenericEventParseConfig, ) -> Vec> { - if inner_instruction.data.len() < 16 { + // Use SIMD-optimized data validation + if !SimdUtils::validate_instruction_data_simd(&inner_instruction.data, 16, 0) { return Vec::new(); } let data = &inner_instruction.data[16..]; @@ -720,80 +837,115 @@ impl EventParser for GenericEventParser { if !self.should_handle(&program_id) { return Ok(()); } - for (disc, configs) in &self.instruction_configs { - if instruction.data.len() < disc.len() { - continue; - } - let discriminator = &instruction.data[..disc.len()]; - let data = &instruction.data[disc.len()..]; - if discriminator == disc { - // 验证账户索引 - if !validate_account_indices(&instruction.accounts, accounts.len()) { - continue; - } - let account_pubkeys: Vec = - instruction.accounts.iter().map(|&idx| accounts[idx as usize]).collect(); - for config in configs { - if config.program_id != program_id { - continue; - } - if let Some(mut event) = self.parse_instruction_event( - config, - data, - &account_pubkeys, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - transaction_index, - ) { - let mut inner_instruction_event: Option> = None; - if inner_instructions.is_some() { - // 解析对应的内部 log 执行 - for inner_instruction in inner_instructions.unwrap().instructions.iter() - { - let result = self.parse_events_from_inner_instruction( - &inner_instruction.instruction, - signature, - slot, - block_time, - program_received_time_us, - outer_index, - inner_index, - transaction_index, - config, - ); - if result.len() > 0 { - inner_instruction_event = Some(result[0].clone()); - } - // 解析swap数据 - let swap_data = parse_swap_data_from_next_instructions( - &*event, - inner_instructions.unwrap(), - inner_index.unwrap_or(-1_i64) as i8, - &accounts, - ); - if let Some(swap_data) = swap_data { - event.set_swap_data(swap_data); - } + // 一维化并行处理：将所有 (discriminator, config) 组合展开并行处理 + let all_processing_params: Vec<_> = self + .instruction_configs + .iter() + .filter(|(disc, _)| { + // Use SIMD-optimized data validation and discriminator matching + SimdUtils::validate_instruction_data_simd(&instruction.data, disc.len(), disc.len()) + && SimdUtils::fast_discriminator_match(&instruction.data, disc) + }) + .flat_map(|(disc, configs)| { + configs + .iter() + .filter(|config| config.program_id == program_id) + .map(move |config| (disc, config)) + }) + .collect(); + + // Use SIMD-optimized account indices validation (只需检查一次) + if !SimdUtils::validate_account_indices_simd(&instruction.accounts, accounts.len()) { + return Ok(()); + } + + // 使用缓存构建账户公钥列表，避免重复分配 (只需构建一次) + let account_pubkeys = { + let mut cache_guard = self.account_cache.lock(); + cache_guard.build_account_pubkeys(&instruction.accounts, accounts).to_vec() + }; + + // 并行处理所有 (discriminator, config) 组合 + let all_results: Vec<_> = all_processing_params + .iter() + .filter_map(|(disc, config)| { + let data = &instruction.data[disc.len()..]; + self.parse_instruction_event( + config, + data, + &account_pubkeys, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + ) + .map(|event| ((*disc).clone(), (*config).clone(), event)) + }) + .collect(); + + for (_disc, config, mut event) in all_results { + // 阻塞处理：原有的同步逻辑 + let mut inner_instruction_event: Option> = None; + if inner_instructions.is_some() { + let inner_instructions_ref = inner_instructions.unwrap(); + + // 并行执行两个任务 + let (inner_event_result, swap_data_result) = std::thread::scope(|s| { + let inner_event_handle = s.spawn(|| { + for inner_instruction in inner_instructions_ref.instructions.iter() { + let result = self.parse_events_from_inner_instruction( + &inner_instruction.instruction, + signature, + slot, + block_time, + program_received_time_us, + outer_index, + inner_index, + transaction_index, + &config, + ); + if result.len() > 0 { + return Some(result[0].clone()); } } - // 合并事件 - if let Some(inner_instruction_event) = inner_instruction_event { - event.merge(&*inner_instruction_event); + None + }); + + let swap_data_handle = s.spawn(|| { + if !event.swap_data_is_parsed() { + parse_swap_data_from_next_instructions( + &*event, + inner_instructions_ref, + inner_index.unwrap_or(-1_i64) as i8, + &accounts, + ) + } else { + None } - // 设置处理时间 - event.set_program_handle_time_consuming_us( - chrono::Utc::now().timestamp_micros() - program_received_time_us, - ); - event = process_event(event, bot_wallet); - callback(&event); - break; - } + }); + + // 等待两个任务完成 + (inner_event_handle.join().unwrap(), swap_data_handle.join().unwrap()) + }); + + inner_instruction_event = inner_event_result; + if let Some(swap_data) = swap_data_result { + event.set_swap_data(swap_data); } } + // 合并事件 + if let Some(inner_instruction_event) = inner_instruction_event { + event.merge(&*inner_instruction_event); + } + // 设置处理时间（使用高性能时钟） + event.set_program_handle_time_consuming_us( + get_high_perf_clock().elapsed_micros_since(program_received_time_us), + ); + event = process_event(event, bot_wallet); + callback(&event); } Ok(()) } @@ -811,11 +963,11 @@ fn process_event( mut event: Box, bot_wallet: Option, ) -> Box { - update_slot(event.slot()); + let slot = event.slot(); if let Some(token_info) = event.as_any().downcast_ref::() { - add_dev_address(token_info.user); + add_dev_address(slot, token_info.user); if token_info.creator != Pubkey::default() && token_info.creator != token_info.user { - add_dev_address(token_info.creator); + add_dev_address(slot, token_info.creator); } } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { if is_dev_address(&trade_info.user) || is_dev_address(&trade_info.creator) { @@ -844,7 +996,7 @@ fn process_event( trade_info.user_quote_amount_out; } } else if let Some(pool_info) = event.as_any().downcast_ref::() { - add_bonk_dev_address(pool_info.creator); + add_bonk_dev_address(slot, pool_info.creator); } else if let Some(trade_info) = event.as_any_mut().downcast_mut::() { if is_bonk_dev_address(&trade_info.payer) { trade_info.is_dev_create_token_trade = true; @@ -854,6 +1006,5 @@ fn process_event( trade_info.is_dev_create_token_trade = false; } } - event.clear_id(); event } diff --git a/src/streaming/event_parser/protocols/block/block_meta_event.rs b/src/streaming/event_parser/protocols/block/block_meta_event.rs index d204eef..8fe4051 100644 --- a/src/streaming/event_parser/protocols/block/block_meta_event.rs +++ b/src/streaming/event_parser/protocols/block/block_meta_event.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use crate::impl_unified_event; use crate::streaming::event_parser::common::{types::EventType, EventMetadata}; use borsh::BorshDeserialize; @@ -20,8 +22,7 @@ impl BlockMetaEvent { program_received_time_us: i64, ) -> Self { let metadata = EventMetadata::new( - format!("block_{}_{}", slot, block_hash), - "".to_string(), + Cow::Borrowed(""), slot, block_time_ms / 1000, block_time_ms, diff --git a/src/streaming/event_parser/protocols/bonk/parser.rs b/src/streaming/event_parser/protocols/bonk/parser.rs index 0c08216..e16fc8f 100755 --- a/src/streaming/event_parser/protocols/bonk/parser.rs +++ b/src/streaming/event_parser/protocols/bonk/parser.rs @@ -118,8 +118,6 @@ impl BonkEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = bonk_pool_create_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); Some(Box::new(BonkPoolCreateEvent { metadata, ..event })) } else { None @@ -132,8 +130,6 @@ impl BonkEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = bonk_trade_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, event.pool_state)); if metadata.event_type == EventType::BonkBuyExactIn || metadata.event_type == EventType::BonkBuyExactOut { @@ -166,9 +162,6 @@ impl BonkEventParser { let minimum_amount_out = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_in, @@ -207,9 +200,6 @@ impl BonkEventParser { let maximum_amount_in = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_out, @@ -248,9 +238,6 @@ impl BonkEventParser { let minimum_amount_out = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_in, @@ -289,9 +276,6 @@ impl BonkEventParser { let maximum_amount_in = read_u64_le(data, 8)?; let share_fee_rate = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}", metadata.signature, accounts[4])); - Some(Box::new(BonkTradeEvent { metadata, amount_out, @@ -332,9 +316,6 @@ impl BonkEventParser { let curve_param = Self::parse_curve_params(data, &mut offset)?; let vesting_param = Self::parse_vesting_params(data, &mut offset)?; - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkPoolCreateEvent { metadata, payer: accounts[0], @@ -369,9 +350,6 @@ impl BonkEventParser { let vesting_param = Self::parse_vesting_params(data, &mut offset)?; let amm_fee_on = data[offset]; - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkPoolCreateEvent { metadata, payer: accounts[0], @@ -514,9 +492,6 @@ impl BonkEventParser { let quote_lot_size = u64::from_le_bytes(data[8..16].try_into().unwrap()); let market_vault_signer_nonce = data[16]; - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkMigrateToAmmEvent { metadata, base_lot_size, @@ -564,9 +539,6 @@ impl BonkEventParser { accounts: &[Pubkey], metadata: EventMetadata, ) -> Option> { - let mut metadata = metadata; - metadata.set_id(metadata.signature.to_string()); - Some(Box::new(BonkMigrateToCpswapEvent { metadata, payer: accounts[0], diff --git a/src/streaming/event_parser/protocols/pumpfun/parser.rs b/src/streaming/event_parser/protocols/pumpfun/parser.rs index 5596f38..0de1634 100755 --- a/src/streaming/event_parser/protocols/pumpfun/parser.rs +++ b/src/streaming/event_parser/protocols/pumpfun/parser.rs @@ -81,8 +81,6 @@ impl PumpFunEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pumpfun_migrate_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, event.user, event.mint)); Some(Box::new(PumpFunMigrateEvent { metadata, ..event })) } else { None @@ -95,11 +93,6 @@ impl PumpFunEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pumpfun_create_token_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.name, event.symbol, event.mint - )); Some(Box::new(PumpFunCreateTokenEvent { metadata, ..event })) } else { None @@ -112,11 +105,6 @@ impl PumpFunEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pumpfun_trade_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.mint, event.user, event.is_buy - )); Some(Box::new(PumpFunTradeEvent { metadata, ..event })) } else { None @@ -151,9 +139,6 @@ impl PumpFunEventParser { Pubkey::default() }; - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, name, symbol, accounts[0])); - Some(Box::new(PumpFunCreateTokenEvent { metadata, name: name.to_string(), @@ -180,8 +165,6 @@ impl PumpFunEventParser { } let amount = u64::from_le_bytes(data[0..8].try_into().unwrap()); let max_sol_cost = u64::from_le_bytes(data[8..16].try_into().unwrap()); - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], true)); Some(Box::new(PumpFunTradeEvent { metadata, global: accounts[0], @@ -216,9 +199,6 @@ impl PumpFunEventParser { } let amount = u64::from_le_bytes(data[0..8].try_into().unwrap()); let min_sol_output = u64::from_le_bytes(data[8..16].try_into().unwrap()); - let mut metadata = metadata; - metadata - .set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], false)); Some(Box::new(PumpFunTradeEvent { metadata, global: accounts[0], @@ -251,8 +231,6 @@ impl PumpFunEventParser { if accounts.len() < 24 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[5], accounts[2])); Some(Box::new(PumpFunMigrateEvent { metadata, global: accounts[0], diff --git a/src/streaming/event_parser/protocols/pumpswap/parser.rs b/src/streaming/event_parser/protocols/pumpswap/parser.rs index dc22336..d50372e 100755 --- a/src/streaming/event_parser/protocols/pumpswap/parser.rs +++ b/src/streaming/event_parser/protocols/pumpswap/parser.rs @@ -91,11 +91,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_buy_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.user, event.pool, event.base_amount_out - )); Some(Box::new(PumpSwapBuyEvent { metadata, ..event })) } else { None @@ -108,11 +103,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_sell_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.user, event.pool, event.base_amount_in - )); Some(Box::new(PumpSwapSellEvent { metadata, ..event })) } else { None @@ -125,11 +115,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_create_pool_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.pool, event.creator, event.base_amount_in - )); Some(Box::new(PumpSwapCreatePoolEvent { metadata, ..event })) } else { None @@ -142,11 +127,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_deposit_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.pool, event.user, event.lp_token_amount_out - )); Some(Box::new(PumpSwapDepositEvent { metadata, ..event })) } else { None @@ -159,11 +139,6 @@ impl PumpSwapEventParser { metadata: EventMetadata, ) -> Option> { if let Some(event) = pump_swap_withdraw_event_log_decode(data) { - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, event.pool, event.user, event.lp_token_amount_in - )); Some(Box::new(PumpSwapWithdrawEvent { metadata, ..event })) } else { None @@ -183,12 +158,6 @@ impl PumpSwapEventParser { let base_amount_out = read_u64_le(data, 0)?; let max_quote_amount_in = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[1], accounts[0], base_amount_out - )); - Some(Box::new(PumpSwapBuyEvent { metadata, base_amount_out, @@ -224,12 +193,6 @@ impl PumpSwapEventParser { let base_amount_in = read_u64_le(data, 0)?; let min_quote_amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[1], accounts[0], base_amount_in - )); - Some(Box::new(PumpSwapSellEvent { metadata, base_amount_in, @@ -271,12 +234,6 @@ impl PumpSwapEventParser { Pubkey::default() }; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[0], accounts[2], base_amount_in - )); - Some(Box::new(PumpSwapCreatePoolEvent { metadata, index, @@ -311,12 +268,6 @@ impl PumpSwapEventParser { let max_base_amount_in = u64::from_le_bytes(data[8..16].try_into().ok()?); let max_quote_amount_in = u64::from_le_bytes(data[16..24].try_into().ok()?); - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[0], accounts[2], lp_token_amount_out - )); - Some(Box::new(PumpSwapDepositEvent { metadata, lp_token_amount_out, @@ -349,12 +300,6 @@ impl PumpSwapEventParser { let min_base_amount_out = u64::from_le_bytes(data[8..16].try_into().ok()?); let min_quote_amount_out = u64::from_le_bytes(data[16..24].try_into().ok()?); - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[0], accounts[2], lp_token_amount_in - )); - Some(Box::new(PumpSwapWithdrawEvent { metadata, lp_token_amount_in, diff --git a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs index 6b8a7ba..f9124e3 100755 --- a/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs @@ -102,12 +102,6 @@ impl RaydiumAmmV4EventParser { return None; } - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4WithdrawPnlEvent { metadata, token_program: accounts[0], @@ -141,12 +135,6 @@ impl RaydiumAmmV4EventParser { } let amount = read_u64_le(data, 0)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4WithdrawEvent { metadata, amount, @@ -190,12 +178,6 @@ impl RaydiumAmmV4EventParser { let init_pc_amount = read_u64_le(data, 9)?; let init_coin_amount = read_u64_le(data, 17)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4Initialize2Event { metadata, nonce, @@ -240,12 +222,6 @@ impl RaydiumAmmV4EventParser { let max_pc_amount = read_u64_le(data, 8)?; let base_side = read_u64_le(data, 16)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumAmmV4DepositEvent { metadata, max_coin_amount, @@ -281,12 +257,6 @@ impl RaydiumAmmV4EventParser { let max_amount_in = read_u64_le(data, 0)?; let amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - let mut accounts = accounts.to_vec(); if accounts.len() == 17 { // 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符 @@ -334,12 +304,6 @@ impl RaydiumAmmV4EventParser { let amount_in = read_u64_le(data, 0)?; let minimum_amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - let mut accounts = accounts.to_vec(); if accounts.len() == 17 { // 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符 diff --git a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs index 7f56274..ced7677 100755 --- a/src/streaming/event_parser/protocols/raydium_clmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_clmm/parser.rs @@ -124,8 +124,6 @@ impl RaydiumClmmEventParser { if data.len() < 51 || accounts.len() < 22 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmOpenPositionV2Event { metadata, tick_lower_index: read_i32_le(data, 0)?, @@ -172,8 +170,6 @@ impl RaydiumClmmEventParser { if data.len() < 51 || accounts.len() < 20 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmOpenPositionWithToken22NftEvent { metadata, tick_lower_index: read_i32_le(data, 0)?, @@ -217,8 +213,6 @@ impl RaydiumClmmEventParser { if data.len() < 34 || accounts.len() < 15 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmIncreaseLiquidityV2Event { metadata, liquidity: read_u128_le(data, 0)?, @@ -252,8 +246,6 @@ impl RaydiumClmmEventParser { if data.len() < 24 || accounts.len() < 13 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmCreatePoolEvent { metadata, sqrt_price_x64: read_u128_le(data, 0)?, @@ -283,8 +275,6 @@ impl RaydiumClmmEventParser { if data.len() < 32 || accounts.len() < 16 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmDecreaseLiquidityV2Event { metadata, liquidity: read_u128_le(data, 0)?, @@ -319,8 +309,6 @@ impl RaydiumClmmEventParser { if accounts.len() < 6 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumClmmClosePositionEvent { metadata, nft_owner: accounts[0], @@ -347,12 +335,6 @@ impl RaydiumClmmEventParser { let sqrt_price_limit_x64 = read_u128_le(data, 16)?; let is_base_input = read_u8_le(data, 32)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[2], accounts[3], accounts[4] - )); - Some(Box::new(RaydiumClmmSwapEvent { metadata, amount, @@ -387,12 +369,6 @@ impl RaydiumClmmEventParser { let sqrt_price_limit_x64 = read_u128_le(data, 16)?; let is_base_input = read_u8_le(data, 32)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[2], accounts[3], accounts[4] - )); - Some(Box::new(RaydiumClmmSwapV2Event { metadata, amount, diff --git a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs index f7d2e2e..d89f779 100755 --- a/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs +++ b/src/streaming/event_parser/protocols/raydium_cpmm/parser.rs @@ -92,8 +92,6 @@ impl RaydiumCpmmEventParser { if data.len() < 24 || accounts.len() < 14 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumCpmmWithdrawEvent { metadata, lp_token_amount: read_u64_le(data, 0)?, @@ -125,8 +123,6 @@ impl RaydiumCpmmEventParser { if data.len() < 24 || accounts.len() < 20 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumCpmmInitializeEvent { metadata, init_amount0: read_u64_le(data, 0)?, @@ -164,8 +160,6 @@ impl RaydiumCpmmEventParser { if data.len() < 24 || accounts.len() < 13 { return None; } - let mut metadata = metadata; - metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1])); Some(Box::new(RaydiumCpmmDepositEvent { metadata, lp_token_amount: read_u64_le(data, 0)?, @@ -200,12 +194,6 @@ impl RaydiumCpmmEventParser { let amount_in = read_u64_le(data, 0)?; let minimum_amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumCpmmSwapEvent { metadata, amount_in, @@ -239,12 +227,6 @@ impl RaydiumCpmmEventParser { let max_amount_in = read_u64_le(data, 0)?; let amount_out = read_u64_le(data, 8)?; - let mut metadata = metadata; - metadata.set_id(format!( - "{}-{}-{}-{}", - metadata.signature, accounts[3], accounts[10], accounts[11] - )); - Some(Box::new(RaydiumCpmmSwapEvent { metadata, max_amount_in, diff --git a/src/streaming/grpc/subscription.rs b/src/streaming/grpc/subscription.rs index 15579fa..06d4d59 100644 --- a/src/streaming/grpc/subscription.rs +++ b/src/streaming/grpc/subscription.rs @@ -14,7 +14,7 @@ use crate::common::AnyResult; use crate::streaming::common::StreamClientConfig as ClientConfig; use crate::streaming::event_parser::common::filter::EventTypeFilter; -/// 订阅管理器 +/// Subscription manager #[derive(Clone)] pub struct SubscriptionManager { endpoint: String, @@ -23,12 +23,12 @@ pub struct SubscriptionManager { } impl SubscriptionManager { - /// 创建新的订阅管理器 + /// Create a new subscription manager pub fn new(endpoint: String, x_token: Option, config: ClientConfig) -> Self { Self { endpoint, x_token, config } } - /// 创建 gRPC 连接 + /// Create gRPC connection pub async fn connect(&self) -> AnyResult> { let builder = GeyserGrpcClient::build_from_shared(self.endpoint.clone())? .x_token(self.x_token.clone())? @@ -39,20 +39,20 @@ impl SubscriptionManager { Ok(builder.connect().await?) } - /// 创建订阅请求并返回流 + /// Create subscription request and return stream pub async fn subscribe_with_request( &self, transactions: Option, accounts: Option, commitment: Option, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> AnyResult<( impl Sink, impl Stream>, SubscribeRequest, )> { let blocks_meta = if event_type_filter.is_some() - && event_type_filter.as_ref().unwrap().include_block_event() + && event_type_filter.unwrap().include_block_event() { hashmap! { "".to_owned() => SubscribeRequestFilterBlocksMeta {} } } else if event_type_filter.is_none() { @@ -76,18 +76,18 @@ impl SubscriptionManager { Ok((sink, stream, subscribe_request)) } - /// 创建账户订阅请求并返回流 + /// Create account subscription request and return stream pub fn subscribe_with_account_request( &self, account: Vec, owner: Vec, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> Option { if account.len() == 0 && owner.len() == 0 { return None; } if event_type_filter.is_some() - && !event_type_filter.as_ref().unwrap().include_account_event() + && !event_type_filter.unwrap().include_account_event() { return None; } @@ -104,16 +104,16 @@ impl SubscriptionManager { Some(accounts) } - /// 生成订阅请求过滤器 + /// Generate subscription request filter pub fn get_subscribe_request_filter( &self, account_include: Vec, account_exclude: Vec, account_required: Vec, - event_type_filter: Option, + event_type_filter: Option<&EventTypeFilter>, ) -> Option { if event_type_filter.is_some() - && !event_type_filter.as_ref().unwrap().include_transaction_event() + && !event_type_filter.unwrap().include_transaction_event() { return None; } @@ -132,7 +132,7 @@ impl SubscriptionManager { Some(transactions) } - /// 获取配置 + /// Get configuration pub fn get_config(&self) -> &ClientConfig { &self.config } diff --git a/src/streaming/shred/connection.rs b/src/streaming/shred/connection.rs index 9c6bbe3..951993c 100644 --- a/src/streaming/shred/connection.rs +++ b/src/streaming/shred/connection.rs @@ -59,14 +59,6 @@ impl ShredStreamGrpc { Self::new_with_config(endpoint, StreamClientConfig::low_latency()).await } - /// Creates a new ShredStreamClient with asynchronous processing configuration. - /// - /// This is a convenience method that creates a client optimized for high-volume scenarios - /// with balanced throughput and reliability. See `StreamClientConfig::async_processing()` - /// for detailed configuration information. - pub async fn new_async_processing(endpoint: String) -> AnyResult { - Self::new_with_config(endpoint, StreamClientConfig::async_processing()).await - } /// 获取当前配置 pub fn get_config(&self) -> &StreamClientConfig { diff --git a/src/streaming/shred_stream.rs b/src/streaming/shred_stream.rs index 1ec59af..34101c0 100755 --- a/src/streaming/shred_stream.rs +++ b/src/streaming/shred_stream.rs @@ -62,19 +62,16 @@ impl ShredStreamGrpc { msg.slot, chrono::Utc::now().timestamp_micros(), ); - // 异步执行，不阻塞主流，使用带背压控制的方法 - let processor_clone = event_processor_clone.clone(); - tokio::spawn(async move { - if let Err(e) = processor_clone - .process_shred_transaction_with_metrics( - transaction_with_slot, - bot_wallet, - ) - .await - { - error!("Error handling message: {e:?}"); - } - }); + // 直接处理，背压控制在 EventProcessor 内部处理 + if let Err(e) = event_processor_clone + .process_shred_transaction_with_metrics( + transaction_with_slot, + bot_wallet, + ) + .await + { + error!("Error handling message: {e:?}"); + } } } } diff --git a/src/streaming/yellowstone_grpc.rs b/src/streaming/yellowstone_grpc.rs index 7ffb886..f083b8b 100644 --- a/src/streaming/yellowstone_grpc.rs +++ b/src/streaming/yellowstone_grpc.rs @@ -106,15 +106,6 @@ impl YellowstoneGrpc { Self::new_with_config(endpoint, x_token, StreamClientConfig::low_latency()) } - /// Creates a new YellowstoneGrpcClient with asynchronous processing configuration. - /// - /// This is a convenience method that creates a client optimized for high-volume scenarios - /// with balanced throughput and reliability. See `StreamClientConfig::async_processing()` - /// for detailed configuration information. - pub fn new_async_processing(endpoint: String, x_token: Option) -> AnyResult { - let config = StreamClientConfig::async_processing(); - Self::new_with_config(endpoint, x_token, config) - } /// 获取配置 pub fn get_config(&self) -> &StreamClientConfig { @@ -196,18 +187,18 @@ impl YellowstoneGrpc { transaction_filter.account_include, transaction_filter.account_exclude, transaction_filter.account_required, - event_type_filter.clone(), + event_type_filter.as_ref(), ); let accounts = self.subscription_manager.subscribe_with_account_request( account_filter.account, account_filter.owner, - event_type_filter.clone(), + event_type_filter.as_ref(), ); // 订阅事件 let (mut subscribe_tx, mut stream, subscribe_request) = self .subscription_manager - .subscribe_with_request(transactions, accounts, commitment, event_type_filter.clone()) + .subscribe_with_request(transactions, accounts, commitment, event_type_filter.as_ref()) .await?; // 用 Arc> 包装 subscribe_tx 以支持多线程共享 @@ -224,84 +215,78 @@ impl YellowstoneGrpc { self.config.backpressure.clone(), Some(Arc::new(callback)), ); - let event_processor = Arc::new(event_processor); let stream_handle = tokio::spawn(async move { loop { tokio::select! { message = stream.next() => { match message { Some(Ok(msg)) => { - // 不阻塞地处理消息，使用 tokio::spawn 实现并发 - let event_processor_ref = Arc::clone(&event_processor); - let subscribe_tx_ref = Arc::clone(&subscribe_tx); - tokio::spawn(async move { - let created_at = msg.created_at; - match msg.update_oneof { - Some(UpdateOneof::Account(account)) => { - let account_pretty = AccountPretty::from(account); - log::debug!("Received account: {:?}", account_pretty); - if let Err(e) = event_processor_ref - .process_grpc_event_transaction_with_metrics( - EventPretty::Account(account_pretty), - bot_wallet, - ) - .await - { - error!("Error processing account event: {e:?}"); - } - } - Some(UpdateOneof::BlockMeta(sut)) => { - let block_meta_pretty = - BlockMetaPretty::from((sut, created_at)); - log::debug!("Received block meta: {:?}", block_meta_pretty); - if let Err(e) = event_processor_ref - .process_grpc_event_transaction_with_metrics( - EventPretty::BlockMeta(block_meta_pretty), - bot_wallet, - ) - .await - { - error!("Error processing block meta event: {e:?}"); - } - } - Some(UpdateOneof::Transaction(sut)) => { - let transaction_pretty = - TransactionPretty::from((sut, created_at)); - log::debug!( - "Received transaction: {} at slot {}", - transaction_pretty.signature, - transaction_pretty.slot - ); - if let Err(e) = event_processor_ref - .process_grpc_event_transaction_with_metrics( - EventPretty::Transaction(transaction_pretty), - bot_wallet, - ) - .await - { - error!("Error processing transaction event: {e:?}"); - } - } - Some(UpdateOneof::Ping(_)) => { - // 只在需要时获取锁，并立即释放 - if let Ok(mut tx_guard) = subscribe_tx_ref.try_lock() { - let _ = tx_guard - .send(SubscribeRequest { - ping: Some(SubscribeRequestPing { id: 1 }), - ..Default::default() - }) - .await; - } - log::debug!("service is ping: {}", Local::now()); - } - Some(UpdateOneof::Pong(_)) => { - log::debug!("service is pong: {}", Local::now()); - } - _ => { - log::debug!("Received other message type"); + let created_at = msg.created_at; + match msg.update_oneof { + Some(UpdateOneof::Account(account)) => { + let account_pretty = AccountPretty::from(account); + log::debug!("Received account: {:?}", account_pretty); + if let Err(e) = event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::Account(account_pretty), + bot_wallet, + ) + .await + { + error!("Error processing account event: {e:?}"); } } - }); + Some(UpdateOneof::BlockMeta(sut)) => { + let block_meta_pretty = + BlockMetaPretty::from((sut, created_at)); + log::debug!("Received block meta: {:?}", block_meta_pretty); + if let Err(e) = event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::BlockMeta(block_meta_pretty), + bot_wallet, + ) + .await + { + error!("Error processing block meta event: {e:?}"); + } + } + Some(UpdateOneof::Transaction(sut)) => { + let transaction_pretty = + TransactionPretty::from((sut, created_at)); + log::debug!( + "Received transaction: {} at slot {}", + transaction_pretty.signature, + transaction_pretty.slot + ); + if let Err(e) = event_processor + .process_grpc_event_transaction_with_metrics( + EventPretty::Transaction(transaction_pretty), + bot_wallet, + ) + .await + { + error!("Error processing transaction event: {e:?}"); + } + } + Some(UpdateOneof::Ping(_)) => { + // 只在需要时获取锁，并立即释放 + if let Ok(mut tx_guard) = subscribe_tx.try_lock() { + let _ = tx_guard + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: 1 }), + ..Default::default() + }) + .await; + } + log::debug!("service is ping: {}", Local::now()); + } + Some(UpdateOneof::Pong(_)) => { + log::debug!("service is pong: {}", Local::now()); + } + _ => { + log::debug!("Received other message type"); + } + } } Some(Err(error)) => { error!("Stream error: {error:?}"); diff --git a/src/streaming/yellowstone_sub_system.rs b/src/streaming/yellowstone_sub_system.rs index d7cabcf..030397d 100755 --- a/src/streaming/yellowstone_sub_system.rs +++ b/src/streaming/yellowstone_sub_system.rs @@ -64,17 +64,14 @@ impl YellowstoneGrpc { Some(UpdateOneof::Transaction(sut)) => { let transaction_pretty = TransactionPretty::from((sut, created_at)); let event_pretty = EventPretty::Transaction(transaction_pretty); - let callback_clone = callback.clone(); - tokio::spawn(async move { - if let Err(e) = Self::process_system_transaction( - event_pretty, - &*callback_clone, - ) - .await - { - error!("Error processing transaction: {e:?}"); - } - }); + if let Err(e) = Self::process_system_transaction( + event_pretty, + &*callback, + ) + .await + { + error!("Error processing transaction: {e:?}"); + } } Some(UpdateOneof::Ping(_)) => { let _ = subscribe_tx