gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-17 02:48:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: restructure event handling from trait objects to enum pattern

Browse Source 
- Convert UnifiedEvent from trait object (Box<dyn UnifiedEvent>) to concrete enum type
  - Remove match_event! macro in favor of native match expressions
  - Simplify event metadata access: event.event_type() -> event.metadata().event_type
  - Unify event callback signatures: Fn(Box<dyn UnifiedEvent>) -> Fn(UnifiedEvent)
  - Update all example code to use the new enum-based event system
  - Optimize type system to reduce runtime overhead and improve type safety

  Affected scope:
  - Core event parser and processor
  - All protocol event definitions (PumpFun, PumpSwap, Bonk, Raydium series, etc.)
  - gRPC and Shred streaming modules
  - All example code
This commit is contained in:
ysq
2025-10-10 18:19:44 +08:00
parent 77caa39f0c
commit 312a675a40
43 changed files with 1013 additions and 1839 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/common/config.rs
+5 -71
View File
@@ -1,35 +1,5 @@
use super::constants::*;
/// Backpressure handling strategy
#[derive(Debug, Clone, Copy)]
pub enum BackpressureStrategy {
/// Block and wait (default)
Block,
/// Drop messages
Drop,
}
impl Default for BackpressureStrategy {
fn default() -> Self {
Self::Block
}
}
/// Backpressure configuration
#[derive(Debug, Clone)]
pub struct BackpressureConfig {
/// Channel size (default: 1000)
pub permits: usize,
/// Backpressure handling strategy (default: Block)
pub strategy: BackpressureStrategy,
}
impl Default for BackpressureConfig {
fn default() -> Self {
Self { permits: 3000, strategy: BackpressureStrategy::default() }
}
}
/// Connection configuration
#[derive(Debug, Clone)]
pub struct ConnectionConfig {
@@ -56,57 +26,21 @@ impl Default for ConnectionConfig {
pub struct StreamClientConfig {
/// Connection configuration
pub connection: ConnectionConfig,
/// Backpressure configuration
pub backpressure: BackpressureConfig,
/// Whether performance monitoring is enabled (default: false)
pub enable_metrics: bool,
}
impl Default for StreamClientConfig {
fn default() -> Self {
Self {
connection: ConnectionConfig::default(),
backpressure: BackpressureConfig::default(),
enable_metrics: false,
}
Self { connection: ConnectionConfig::default(), enable_metrics: false }
}
}
impl StreamClientConfig {
/// Creates a high-throughput configuration optimized for high-concurrency scenarios.
///
/// This configuration prioritizes throughput over latency by:
/// - Implementing a drop strategy for backpressure to avoid blocking
/// - Setting a large permit buffer (5,000) to handle burst traffic
///
/// Ideal for scenarios where you need to process large volumes of data
/// and can tolerate occasional message drops during peak loads.
pub fn high_throughput() -> Self {
Self {
connection: ConnectionConfig::default(),
backpressure: BackpressureConfig {
permits: 20000,
strategy: BackpressureStrategy::Drop,
},
enable_metrics: false,
}
}
/// Creates a low-latency configuration optimized for real-time scenarios.
///
/// This configuration prioritizes latency over throughput by:
/// - Processing events immediately without buffering
/// - Implementing a blocking backpressure strategy to ensure no data loss
/// - Setting optimal permits (4000) for balanced throughput and latency
///
/// Ideal for scenarios where every millisecond counts and you cannot
/// afford to lose any events, such as trading applications or real-time monitoring.
pub fn low_latency() -> Self {
Self {
connection: ConnectionConfig::default(),
backpressure: BackpressureConfig { permits: 4000, strategy: BackpressureStrategy::Block },
enable_metrics: false,
}
Self::default()
}
pub fn high_throughput() -> Self {
Self::default()
}
}
src/streaming/common/event_processor.rs
+41 -262
View File
@@ -1,21 +1,16 @@
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use crossbeam_queue::SegQueue;
use solana_sdk::pubkey::Pubkey;
use crate::common::AnyResult;
use crate::streaming::common::BackpressureStrategy;
use crate::streaming::common::{
MetricsEventType, MetricsManager, StreamClientConfig as ClientConfig,
};
use crate::streaming::common::{MetricsEventType, StreamClientConfig as ClientConfig};
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::core::account_event_parser::AccountEventParser;
use crate::streaming::event_parser::core::common_event_parser::CommonEventParser;
use crate::streaming::event_parser::core::event_parser::EventParser;
use crate::streaming::event_parser::{core::traits::UnifiedEvent, Protocol};
use crate::streaming::grpc::{BackpressureConfig, EventPretty};
use crate::streaming::grpc::{EventPretty, MetricsManager};
use crate::streaming::shred::TransactionWithSlot;
use once_cell::sync::OnceCell;
@@ -24,141 +19,54 @@ pub enum EventSource {
Shred,
}
/// High-performance Event processor using SegQueue for all strategies
/// High-performance Event processor
pub struct EventProcessor {
pub(crate) metrics_manager: MetricsManager,
pub(crate) config: ClientConfig,
pub(crate) parser_cache: OnceCell<Arc<EventParser>>,
pub(crate) protocols: Vec<Protocol>,
pub(crate) event_type_filter: Option<EventTypeFilter>,
pub(crate) callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
pub(crate) backpressure_config: BackpressureConfig,
pub(crate) grpc_queue: Arc<SegQueue<(EventPretty, Option<Pubkey>)>>,
pub(crate) shred_queue: Arc<SegQueue<(TransactionWithSlot, Option<Pubkey>)>>,
pub(crate) grpc_pending_count: Arc<AtomicUsize>,
pub(crate) shred_pending_count: Arc<AtomicUsize>,
pub(crate) processing_shutdown: Arc<AtomicBool>,
pub(crate) callback: Option<Arc<dyn Fn(UnifiedEvent) + Send + Sync>>,
}
impl EventProcessor {
pub fn new(metrics_manager: MetricsManager, config: ClientConfig) -> Self {
let backpressure_config = config.backpressure.clone();
let grpc_queue = Arc::new(SegQueue::new());
let shred_queue = Arc::new(SegQueue::new());
let grpc_pending_count = Arc::new(AtomicUsize::new(0));
let shred_pending_count = Arc::new(AtomicUsize::new(0));
let processing_shutdown = Arc::new(AtomicBool::new(false));
pub fn new(config: ClientConfig) -> Self {
Self {
metrics_manager,
config,
parser_cache: OnceCell::new(),
protocols: vec![],
event_type_filter: None,
backpressure_config,
callback: None,
grpc_queue,
shred_queue,
grpc_pending_count,
shred_pending_count,
processing_shutdown,
}
}
pub fn set_protocols_and_event_type_filter(
&mut self,
source: EventSource,
_source: EventSource,
protocols: Vec<Protocol>,
event_type_filter: Option<EventTypeFilter>,
backpressure_config: BackpressureConfig,
callback: Option<Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync>>,
callback: Option<Arc<dyn Fn(UnifiedEvent) + Send + Sync>>,
) {
self.protocols = protocols;
self.event_type_filter = event_type_filter;
self.backpressure_config = backpressure_config;
self.callback = callback;
let protocols_ref = &self.protocols;
let event_type_filter_ref = self.event_type_filter.as_ref();
self.parser_cache.get_or_init(|| {
Arc::new(EventParser::new(protocols_ref.clone(), event_type_filter_ref.cloned()))
});
if matches!(self.backpressure_config.strategy, BackpressureStrategy::Block) {
self.start_block_processing_thread(source);
}
}
pub fn get_parser(&self) -> Arc<EventParser> {
self.parser_cache.get().unwrap().clone()
}
fn create_adapter_callback(&self) -> Arc<dyn Fn(Box<dyn UnifiedEvent>) + Send + Sync> {
let callback = self.callback.clone().unwrap();
let metrics_manager = self.metrics_manager.clone();
Arc::new(move |event: Box<dyn UnifiedEvent>| {
let processing_time_us = event.handle_us() as f64;
fn invoke_callback(&self, event: UnifiedEvent) {
if let Some(callback) = self.callback.as_ref() {
callback(event);
metrics_manager.update_metrics(MetricsEventType::Transaction, 1, processing_time_us);
})
}
pub async fn process_grpc_event_transaction_with_metrics(
&self,
event_pretty: EventPretty,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
self.apply_backpressure_control(event_pretty, bot_wallet).await
}
async fn apply_backpressure_control(
&self,
event_pretty: EventPretty,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
match self.backpressure_config.strategy {
BackpressureStrategy::Block => {
loop {
let current_pending = self.grpc_pending_count.load(Ordering::Relaxed);
if current_pending < self.backpressure_config.permits {
self.grpc_queue.push((event_pretty, bot_wallet));
self.grpc_pending_count.fetch_add(1, Ordering::Relaxed);
break;
}
tokio::task::yield_now().await;
}
Ok(())
}
BackpressureStrategy::Drop => {
let current_pending = self.grpc_pending_count.load(Ordering::Relaxed);
if current_pending >= self.backpressure_config.permits {
self.metrics_manager.increment_dropped_events();
Ok(())
} else {
self.grpc_pending_count.fetch_add(1, Ordering::Relaxed);
let processor = self.clone();
tokio::spawn(async move {
match processor
.process_grpc_event_transaction(event_pretty, bot_wallet)
.await
{
Ok(_) => {
processor.grpc_pending_count.fetch_sub(1, Ordering::Relaxed);
}
Err(e) => {
log::error!("Error in async gRPC processing: {}", e);
}
}
});
Ok(())
}
}
}
}
async fn process_grpc_event_transaction(
pub async fn process_grpc_transaction(
&self,
event_pretty: EventPretty,
bot_wallet: Option<Pubkey>,
@@ -168,20 +76,20 @@ impl EventProcessor {
}
match event_pretty {
EventPretty::Account(account_pretty) => {
self.metrics_manager.add_account_process_count();
MetricsManager::global().add_account_process_count();
let account_event = AccountEventParser::parse_account_event(
&self.protocols,
account_pretty,
self.event_type_filter.as_ref(),
);
if let Some(event) = account_event {
let processing_time_us = event.handle_us() as f64;
if let Some(mut event) = account_event {
let processing_time_us = event.metadata().handle_us as f64;
self.invoke_callback(event);
self.update_metrics(MetricsEventType::Account, 1, processing_time_us);
}
}
EventPretty::Transaction(transaction_pretty) => {
self.metrics_manager.add_tx_process_count();
MetricsManager::global().add_tx_process_count();
let slot = transaction_pretty.slot;
let signature = transaction_pretty.signature;
let block_time = transaction_pretty.block_time;
@@ -190,7 +98,17 @@ impl EventProcessor {
let grpc_tx = transaction_pretty.grpc_tx;
let parser = self.get_parser();
let adapter_callback = self.create_adapter_callback();
let callback = self.callback.clone().unwrap();
let adapter_callback = Arc::new(move |mut event: UnifiedEvent| {
let processing_time_us = event.metadata().handle_us as f64;
callback(event);
MetricsManager::global().update_metrics(
MetricsEventType::Transaction,
1,
processing_time_us,
);
});
parser
.parse_grpc_transaction_owned(
grpc_tx,
@@ -205,18 +123,18 @@ impl EventProcessor {
.await?;
}
EventPretty::BlockMeta(block_meta_pretty) => {
self.metrics_manager.add_block_meta_process_count();
MetricsManager::global().add_block_meta_process_count();
let block_time_ms = block_meta_pretty
.block_time
.map(|ts| ts.seconds * 1000 + ts.nanos as i64 / 1_000_000)
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
let block_meta_event = CommonEventParser::generate_block_meta_event(
let mut block_meta_event = CommonEventParser::generate_block_meta_event(
block_meta_pretty.slot,
block_meta_pretty.block_hash,
block_time_ms,
block_meta_pretty.recv_us,
);
let processing_time_us = block_meta_event.handle_us() as f64;
let processing_time_us = block_meta_event.metadata().handle_us as f64;
self.invoke_callback(block_meta_event);
self.update_metrics(MetricsEventType::BlockMeta, 1, processing_time_us);
}
@@ -225,74 +143,6 @@ impl EventProcessor {
Ok(())
}
pub fn invoke_callback(&self, event: Box<dyn UnifiedEvent>) {
if let Some(callback) = self.callback.as_ref() {
callback(event);
}
}
pub async fn process_shred_transaction_immediate(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
self.process_shred_transaction(transaction_with_slot, bot_wallet).await
}
pub async fn process_shred_transaction_with_metrics(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
self.apply_shred_backpressure_control(transaction_with_slot, bot_wallet).await
}
async fn apply_shred_backpressure_control(
&self,
transaction_with_slot: TransactionWithSlot,
bot_wallet: Option<Pubkey>,
) -> AnyResult<()> {
match self.backpressure_config.strategy {
BackpressureStrategy::Block => {
loop {
let current_pending = self.shred_pending_count.load(Ordering::Relaxed);
if current_pending < self.backpressure_config.permits {
self.shred_queue.push((transaction_with_slot, bot_wallet));
self.shred_pending_count.fetch_add(1, Ordering::Relaxed);
break;
}
tokio::task::yield_now().await;
}
Ok(())
}
BackpressureStrategy::Drop => {
let current_pending = self.shred_pending_count.load(Ordering::Relaxed);
if current_pending >= self.backpressure_config.permits {
self.metrics_manager.increment_dropped_events();
Ok(())
} else {
self.shred_pending_count.fetch_add(1, Ordering::Relaxed);
let processor = self.clone();
tokio::spawn(async move {
match processor
.process_shred_transaction(transaction_with_slot, bot_wallet)
.await
{
Ok(_) => {
processor.shred_pending_count.fetch_sub(1, Ordering::Relaxed);
}
Err(e) => {
log::error!("Error in async shred processing: {}", e);
}
}
});
Ok(())
}
}
}
}
pub async fn process_shred_transaction(
&self,
transaction_with_slot: TransactionWithSlot,
@@ -301,7 +151,7 @@ impl EventProcessor {
if self.callback.is_none() {
return Ok(());
}
self.metrics_manager.add_tx_process_count();
MetricsManager::global().add_tx_process_count();
let tx = transaction_with_slot.transaction;
let slot = transaction_with_slot.slot;
@@ -312,7 +162,17 @@ impl EventProcessor {
let recv_us = transaction_with_slot.recv_us;
let parser = self.get_parser();
let adapter_callback = self.create_adapter_callback();
let callback = self.callback.clone().unwrap();
let adapter_callback = Arc::new(move |mut event: UnifiedEvent| {
let processing_time_us = event.metadata().handle_us as f64;
callback(event);
MetricsManager::global().update_metrics(
MetricsEventType::Transaction,
1,
processing_time_us,
);
});
parser
.parse_versioned_transaction_owned(
tx,
@@ -331,99 +191,18 @@ impl EventProcessor {
}
fn update_metrics(&self, ty: MetricsEventType, count: u64, time_us: f64) {
self.metrics_manager.update_metrics(ty, count, time_us);
}
fn start_block_processing_thread(&self, source: EventSource) {
self.processing_shutdown.store(false, Ordering::Relaxed);
let grpc_queue = Arc::clone(&self.grpc_queue);
let shred_queue = Arc::clone(&self.shred_queue);
let grpc_pending_count = Arc::clone(&self.grpc_pending_count);
let shred_pending_count = Arc::clone(&self.shred_pending_count);
let shutdown_flag = Arc::clone(&self.processing_shutdown);
let shutdown_flag_clone = Arc::clone(&self.processing_shutdown);
let processor = self.clone();
let processor_clone = self.clone();
// Dedicated thread with busy-wait and lock-free processing
match source {
EventSource::Grpc => {
std::thread::spawn(move || {
let worker_threads =
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); // 如果获取失败则回退到4个线程
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(worker_threads)
.enable_all()
.build()
.unwrap();
while !shutdown_flag.load(Ordering::Relaxed) {
if let Some((event_pretty, bot_wallet)) = grpc_queue.pop() {
grpc_pending_count.fetch_sub(1, Ordering::Relaxed);
if let Err(e) = rt.block_on(
processor.process_grpc_event_transaction(event_pretty, bot_wallet),
) {
println!("Error processing gRPC event: {}", e);
}
} else {
// 待测试替换方案： lock-free queue + spin + batch
std::thread::sleep(std::time::Duration::from_micros(500));
}
}
});
}
EventSource::Shred => {
// Shred processing with same low-latency optimization
std::thread::spawn(move || {
let worker_threads =
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4); // 如果获取失败则回退到4个线程
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(worker_threads)
.enable_all()
.build()
.unwrap();
while !shutdown_flag_clone.load(Ordering::Relaxed) {
if let Some((transaction_with_slot, bot_wallet)) = shred_queue.pop() {
shred_pending_count.fetch_sub(1, Ordering::Relaxed);
if let Err(e) = rt.block_on(
processor_clone
.process_shred_transaction(transaction_with_slot, bot_wallet),
) {
log::error!("Error processing shred transaction: {}", e);
}
} else {
// 待测试替换方案： lock-free queue + spin + batch
std::thread::sleep(std::time::Duration::from_micros(500));
}
}
});
}
}
}
pub fn stop_processing(&self) {
self.processing_shutdown.store(true, Ordering::Relaxed);
MetricsManager::global().update_metrics(ty, count, time_us);
}
}
impl Clone for EventProcessor {
fn clone(&self) -> Self {
Self {
metrics_manager: self.metrics_manager.clone(),
config: self.config.clone(),
parser_cache: self.parser_cache.clone(),
protocols: self.protocols.clone(),
event_type_filter: self.event_type_filter.clone(),
backpressure_config: self.backpressure_config.clone(),
callback: self.callback.clone(),
grpc_queue: self.grpc_queue.clone(),
shred_queue: self.shred_queue.clone(),
grpc_pending_count: self.grpc_pending_count.clone(),
shred_pending_count: self.shred_pending_count.clone(),
processing_shutdown: self.processing_shutdown.clone(),
}
}
}
src/streaming/common/metrics.rs
+124 -158
View File
@@ -1,5 +1,4 @@
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use super::constants::*;
@@ -44,13 +43,13 @@ struct AtomicEventMetrics {
}
impl AtomicEventMetrics {
fn new(now_nanos: u64) -> Self {
const fn new_const() -> Self {
Self {
process_count: AtomicU64::new(0),
events_processed: AtomicU64::new(0),
events_in_window: AtomicU64::new(0),
window_start_nanos: AtomicU64::new(now_nanos),
processing_stats: AtomicProcessingTimeStats::new(),
window_start_nanos: AtomicU64::new(0),
processing_stats: AtomicProcessingTimeStats::new_const(),
}
}
@@ -114,43 +113,24 @@ struct AtomicProcessingTimeStats {
}
impl AtomicProcessingTimeStats {
fn new() -> Self {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
const fn new_const() -> Self {
Self {
min_time_bits: AtomicU64::new(f64::INFINITY.to_bits()),
max_time_bits: AtomicU64::new(0),
min_time_timestamp_nanos: AtomicU64::new(now_nanos),
max_time_timestamp_nanos: AtomicU64::new(now_nanos),
min_time_timestamp_nanos: AtomicU64::new(0),
max_time_timestamp_nanos: AtomicU64::new(0),
total_time_us: AtomicU64::new(0),
total_events: AtomicU64::new(0),
}
}
/// Atomically update processing time statistics
/// Atomically update processing time statistics (hot path - no syscalls)
#[inline]
fn update(&self, time_us: f64, event_count: u64) {
let time_bits = time_us.to_bits();
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
// Update minimum value, check time difference and reset if over 10 seconds
// Fast path: Update min value without timestamp check (checked in background task)
let mut current_min = self.min_time_bits.load(Ordering::Relaxed);
let min_timestamp = self.min_time_timestamp_nanos.load(Ordering::Relaxed);
// Check if min value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds)
let min_time_diff_nanos = now_nanos.saturating_sub(min_timestamp);
if min_time_diff_nanos > 10_000_000_000 {
// Over 10 seconds, reset min value
self.min_time_bits.store(f64::INFINITY.to_bits(), Ordering::Relaxed);
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
current_min = f64::INFINITY.to_bits();
}
// If current time is less than min value, update min value and timestamp
while time_bits < current_min {
match self.min_time_bits.compare_exchange_weak(
current_min,
@@ -158,29 +138,13 @@ impl AtomicProcessingTimeStats {
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
// Successfully updated min value, also update timestamp
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
break;
}
Ok(_) => break,
Err(x) => current_min = x,
}
}
// Update maximum value, check time difference and reset if over 10 seconds
// Fast path: Update max value without timestamp check (checked in background task)
let mut current_max = self.max_time_bits.load(Ordering::Relaxed);
let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed);
// Check if max value timestamp exceeds 10 seconds (10_000_000_000 nanoseconds)
let time_diff_nanos = now_nanos.saturating_sub(max_timestamp);
if time_diff_nanos > 10_000_000_000 {
// Over 10 seconds, reset max value
self.max_time_bits.store(0, Ordering::Relaxed);
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
current_max = 0;
}
// If current time is greater than max value, update max value and timestamp
while time_bits > current_max {
match self.max_time_bits.compare_exchange_weak(
current_max,
@@ -188,11 +152,7 @@ impl AtomicProcessingTimeStats {
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
// Successfully updated max value, also update timestamp
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
break;
}
Ok(_) => break,
Err(x) => current_max = x,
}
}
@@ -203,6 +163,28 @@ impl AtomicProcessingTimeStats {
self.total_events.fetch_add(event_count, Ordering::Relaxed);
}
/// Reset min/max if they are older than 10 seconds (called by background task)
#[inline]
fn reset_stale_min_max(&self) {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
// Check and reset min if stale
let min_timestamp = self.min_time_timestamp_nanos.load(Ordering::Relaxed);
if now_nanos.saturating_sub(min_timestamp) > 10_000_000_000 {
self.min_time_bits.store(f64::INFINITY.to_bits(), Ordering::Relaxed);
self.min_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
}
// Check and reset max if stale
let max_timestamp = self.max_time_timestamp_nanos.load(Ordering::Relaxed);
if now_nanos.saturating_sub(max_timestamp) > 10_000_000_000 {
self.max_time_bits.store(0, Ordering::Relaxed);
self.max_time_timestamp_nanos.store(now_nanos, Ordering::Relaxed);
}
}
/// Get statistics (non-blocking)
#[inline]
fn get_stats(&self) -> ProcessingTimeStats {
@@ -275,7 +257,7 @@ impl PerformanceMetrics {
/// High-performance metrics system
#[derive(Debug)]
pub struct HighPerformanceMetrics {
start_nanos: u64,
start_nanos: AtomicU64,
event_metrics: [AtomicEventMetrics; 3],
processing_stats: AtomicProcessingTimeStats,
// 丢弃事件指标
@@ -283,20 +265,16 @@ pub struct HighPerformanceMetrics {
}
impl HighPerformanceMetrics {
fn new() -> Self {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
/// Const constructor for static initialization (zero-cost)
const fn new_const() -> Self {
Self {
start_nanos: now_nanos,
start_nanos: AtomicU64::new(0), // Will be lazily initialized on first access
event_metrics: [
AtomicEventMetrics::new(now_nanos),
AtomicEventMetrics::new(now_nanos),
AtomicEventMetrics::new(now_nanos),
AtomicEventMetrics::new_const(),
AtomicEventMetrics::new_const(),
AtomicEventMetrics::new_const(),
],
processing_stats: AtomicProcessingTimeStats::new(),
// 初始化丢弃事件指标
processing_stats: AtomicProcessingTimeStats::new_const(),
dropped_events_count: AtomicU64::new(0),
}
}
@@ -307,7 +285,23 @@ impl HighPerformanceMetrics {
let now_nanos =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
as u64;
(now_nanos - self.start_nanos) as f64 / 1_000_000_000.0
// Lazy initialization of start_nanos (compare-and-swap once)
let mut start = self.start_nanos.load(Ordering::Relaxed);
if start == 0 {
// Try to initialize with current time
match self.start_nanos.compare_exchange(
0,
now_nanos,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => start = now_nanos,
Err(existing) => start = existing,
}
}
(now_nanos - start) as f64 / 1_000_000_000.0
}
/// 获取事件指标快照
@@ -348,122 +342,122 @@ impl HighPerformanceMetrics {
}
}
/// 高性能指标管理器
pub struct MetricsManager {
metrics: Arc<HighPerformanceMetrics>,
enable_metrics: bool,
stream_name: String,
background_task_running: AtomicBool,
}
/// Global singleton instance - zero-cost static allocation
static GLOBAL_METRICS: HighPerformanceMetrics = HighPerformanceMetrics::new_const();
/// Background task initialization flag
static BACKGROUND_TASK_STARTED: AtomicBool = AtomicBool::new(false);
/// Metrics enabled flag
static METRICS_ENABLED: AtomicBool = AtomicBool::new(true);
/// 高性能指标管理器 (Singleton)
#[derive(Clone, Copy)]
pub struct MetricsManager;
impl MetricsManager {
/// 创建新的指标管理器
pub fn new(enable_metrics: bool, stream_name: String) -> Self {
let manager = Self {
metrics: Arc::new(HighPerformanceMetrics::new()),
enable_metrics,
stream_name,
background_task_running: AtomicBool::new(false),
};
// 启动后台任务
manager.start_background_tasks();
manager
/// Get global singleton instance (zero-cost)
#[inline]
pub const fn global() -> Self {
Self
}
/// 启动后台任务
fn start_background_tasks(&self) {
if self
.background_task_running
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
/// Initialize and start background task (call once at startup)
pub fn init(enable_metrics: bool) {
METRICS_ENABLED.store(enable_metrics, Ordering::Relaxed);
// Start background task only once
if enable_metrics
&& BACKGROUND_TASK_STARTED
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
if !self.enable_metrics {
return;
}
let metrics = self.metrics.clone();
tokio::spawn(async move {
tokio::spawn(async {
let mut interval = tokio::time::interval(std::time::Duration::from_millis(500));
loop {
interval.tick().await;
let window_duration_nanos = DEFAULT_METRICS_WINDOW_SECONDS * 1_000_000_000;
// 更新所有事件类型的窗口指标
metrics.update_window_metrics(EventType::Transaction, window_duration_nanos);
metrics.update_window_metrics(EventType::Account, window_duration_nanos);
metrics.update_window_metrics(EventType::BlockMeta, window_duration_nanos);
// Update window metrics for all event types
GLOBAL_METRICS
.update_window_metrics(EventType::Transaction, window_duration_nanos);
GLOBAL_METRICS.update_window_metrics(EventType::Account, window_duration_nanos);
GLOBAL_METRICS
.update_window_metrics(EventType::BlockMeta, window_duration_nanos);
// Reset stale min/max values (10 second expiry) - moved from hot path
GLOBAL_METRICS.processing_stats.reset_stale_min_max();
for event_metric in &GLOBAL_METRICS.event_metrics {
event_metric.processing_stats.reset_stale_min_max();
}
}
});
}
}
#[inline]
fn is_enabled(&self) -> bool {
METRICS_ENABLED.load(Ordering::Relaxed)
}
/// 记录处理次数（非阻塞）
#[inline]
pub fn record_process(&self, event_type: EventType) {
if self.enable_metrics {
self.metrics.event_metrics[event_type.as_index()].add_process_count();
if self.is_enabled() {
GLOBAL_METRICS.event_metrics[event_type.as_index()].add_process_count();
}
}
/// 记录事件处理（非阻塞）
#[inline]
pub fn record_events(&self, event_type: EventType, count: u64, processing_time_us: f64) {
if !self.enable_metrics {
if !self.is_enabled() {
return;
}
let index = event_type.as_index();
// 原子更新事件计数
self.metrics.event_metrics[index].add_events_processed(count);
GLOBAL_METRICS.event_metrics[index].add_events_processed(count);
// 原子更新该事件类型的处理时间统计
self.metrics.event_metrics[index].update_processing_stats(processing_time_us, count);
GLOBAL_METRICS.event_metrics[index].update_processing_stats(processing_time_us, count);
// 保持全局处理时间统计的兼容性
self.metrics.processing_stats.update(processing_time_us, count);
GLOBAL_METRICS.processing_stats.update(processing_time_us, count);
}
/// 记录慢处理操作
#[inline]
pub fn log_slow_processing(&self, processing_time_us: f64, event_count: usize) {
if processing_time_us > SLOW_PROCESSING_THRESHOLD_US {
log::debug!(
"{} slow processing: {:.2}us for {} events",
self.stream_name,
processing_time_us,
event_count,
);
log::debug!("Slow processing: {:.2}us for {} events", processing_time_us, event_count);
}
}
/// 获取运行时长
pub fn get_uptime(&self) -> std::time::Duration {
std::time::Duration::from_secs_f64(self.metrics.get_uptime_seconds())
std::time::Duration::from_secs_f64(GLOBAL_METRICS.get_uptime_seconds())
}
/// 获取事件指标
pub fn get_event_metrics(&self, event_type: EventType) -> EventMetricsSnapshot {
self.metrics.get_event_metrics(event_type)
GLOBAL_METRICS.get_event_metrics(event_type)
}
/// 获取处理时间统计
pub fn get_processing_stats(&self) -> ProcessingTimeStats {
self.metrics.get_processing_stats()
GLOBAL_METRICS.get_processing_stats()
}
/// 获取丢弃事件计数
pub fn get_dropped_events_count(&self) -> u64 {
self.metrics.get_dropped_events_count()
GLOBAL_METRICS.get_dropped_events_count()
}
/// 打印性能指标（非阻塞）
pub fn print_metrics(&self) {
println!("\n📊 {} Performance Metrics", self.stream_name);
println!("\n📊 Performance Metrics");
println!(" Run Time: {:?}", self.get_uptime());
// 打印丢弃事件指标
@@ -496,34 +490,22 @@ impl MetricsManager {
/// 启动自动性能监控任务
pub async fn start_auto_monitoring(&self) -> Option<tokio::task::JoinHandle<()>> {
if !self.enable_metrics {
if !self.is_enabled() {
return None;
}
let manager = self.clone();
let handle = tokio::spawn(async move {
let handle = tokio::spawn(async {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(
DEFAULT_METRICS_PRINT_INTERVAL_SECONDS,
));
loop {
interval.tick().await;
manager.print_metrics();
MetricsManager::global().print_metrics();
}
});
Some(handle)
}
// === 兼容性方法 ===
/// 兼容性构造函数
pub fn new_with_metrics(
_metrics: Arc<std::sync::RwLock<PerformanceMetrics>>,
enable_metrics: bool,
stream_name: String,
) -> Self {
Self::new(enable_metrics, stream_name)
}
/// 获取完整的性能指标（兼容性方法）
pub fn get_metrics(&self) -> PerformanceMetrics {
PerformanceMetrics {
@@ -532,7 +514,7 @@ impl MetricsManager {
account_metrics: self.get_event_metrics(EventType::Account),
block_meta_metrics: self.get_event_metrics(EventType::BlockMeta),
processing_stats: self.get_processing_stats(),
dropped_events_count: self.metrics.get_dropped_events_count(),
dropped_events_count: self.get_dropped_events_count(),
}
}
@@ -569,54 +551,38 @@ impl MetricsManager {
/// 增加丢弃事件计数
#[inline]
pub fn increment_dropped_events(&self) {
if !self.enable_metrics {
if !self.is_enabled() {
return;
}
// 原子地增加丢弃事件计数
let new_count = self.metrics.dropped_events_count.fetch_add(1, Ordering::Relaxed) + 1;
let new_count = GLOBAL_METRICS.dropped_events_count.fetch_add(1, Ordering::Relaxed) + 1;
// 每丢弃1000个事件记录一次警告日志
if new_count % 1000 == 0 {
log::debug!("{} dropped events count reached: {}", self.stream_name, new_count);
log::debug!("Dropped events count reached: {}", new_count);
}
}
/// 批量增加丢弃事件计数
#[inline]
pub fn increment_dropped_events_by(&self, count: u64) {
if !self.enable_metrics || count == 0 {
if !self.is_enabled() || count == 0 {
return;
}
// 原子地增加丢弃事件计数
let new_count =
self.metrics.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count;
GLOBAL_METRICS.dropped_events_count.fetch_add(count, Ordering::Relaxed) + count;
// 记录批量丢弃事件的日志
if count > 1 {
log::debug!(
"{} dropped batch of {} events, total dropped: {}",
self.stream_name,
count,
new_count
);
log::debug!("Dropped batch of {} events, total dropped: {}", count, new_count);
}
// 每丢弃1000个事件记录一次警告日志
if new_count % 1000 == 0 || (new_count / 1000) != ((new_count - count) / 1000) {
log::debug!("{} dropped events count reached: {}", self.stream_name, new_count);
}
}
}
impl Clone for MetricsManager {
fn clone(&self) -> Self {
Self {
metrics: self.metrics.clone(),
enable_metrics: self.enable_metrics,
stream_name: self.stream_name.clone(),
background_task_running: AtomicBool::new(false), // 新实例不自动启动后台任务
log::debug!("Dropped events count reached: {}", new_count);
}
}
}