gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-19 11:58:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: add object pooling and enhanced metrics for streaming optimization

Browse Source 
- Implement gRPC/shred connection pools for memory efficiency
- Add atomic processing time stats with auto-calibration clock
- Optimize event parsers and protocol handlers
- Refactor metrics system for advanced performance monitoring
This commit is contained in:
ysq
2025-09-02 17:27:27 +08:00
parent 07bb110dc0
commit b71a83bf66
30 changed files with 920 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/event_parser/core/account_event_parser.rs
+5 -7
View File
@@ -1,4 +1,3 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::OnceLock;
@@ -7,7 +6,7 @@ use solana_sdk::pubkey::Pubkey;
use crate::streaming::common::SimdUtils;
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::common::{EventMetadata, EventType, ProtocolType};
use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock};
use crate::streaming::event_parser::core::traits::{elapsed_micros_since, UnifiedEvent};
use crate::streaming::event_parser::protocols::bonk::parser::BONK_PROGRAM_ID;
use crate::streaming::event_parser::protocols::pumpfun::parser::PUMPFUN_PROGRAM_ID;
use crate::streaming::event_parser::protocols::pumpswap::parser::PUMPSWAP_PROGRAM_ID;
@@ -177,12 +176,11 @@ impl AccountEventParser {
if account.owner == config.program_id
&& SimdUtils::fast_discriminator_match(&account.data, config.account_discriminator)
{
let signature_str = Cow::Owned(account.signature.to_string());
let event = (config.account_parser)(
&account,
EventMetadata {
slot: account.slot,
signature: signature_str,
signature: account.signature,
protocol: config.protocol_type,
event_type: config.event_type,
program_id: config.program_id,
@@ -191,9 +189,9 @@ impl AccountEventParser {
},
);
if let Some(mut event) = event {
event.set_program_handle_time_consuming_us(
get_high_perf_clock().elapsed_micros_since(account.program_received_time_us),
);
event.set_program_handle_time_consuming_us(elapsed_micros_since(
account.program_received_time_us,
));
return Some(event);
}
}
src/streaming/event_parser/core/common_event_parser.rs
+6 -11
View File
@@ -1,4 +1,4 @@
use crate::streaming::event_parser::core::traits::{UnifiedEvent, get_high_perf_clock};
use crate::streaming::event_parser::core::traits::{elapsed_micros_since, UnifiedEvent};
use crate::streaming::event_parser::protocols::block::block_meta_event::BlockMetaEvent;
pub struct CommonEventParser {}
@@ -6,19 +6,14 @@ pub struct CommonEventParser {}
impl CommonEventParser {
pub fn generate_block_meta_event(
slot: u64,
block_hash: &str,
block_hash: String,
block_time_ms: i64,
program_received_time_us: i64,
) -> Box<dyn UnifiedEvent> {
let mut block_meta_event = BlockMetaEvent::new(
slot,
block_hash.to_string(),
block_time_ms,
program_received_time_us,
);
block_meta_event.set_program_handle_time_consuming_us(
get_high_perf_clock().elapsed_micros_since(program_received_time_us),
);
let mut block_meta_event =
BlockMetaEvent::new(slot, block_hash, block_time_ms, program_received_time_us);
block_meta_event
.set_program_handle_time_consuming_us(elapsed_micros_since(program_received_time_us));
Box::new(block_meta_event)
}
}
src/streaming/event_parser/core/traits.rs
+95 -17
View File
@@ -29,22 +29,53 @@ use crate::streaming::event_parser::{
},
};
/// 高性能时钟管理器，减少系统调用开销
/// 高性能时钟管理器，减少系统调用开销并最小化延迟
#[derive(Debug)]
pub struct HighPerformanceClock {
/// 基准时间点（程序启动时的单调时钟时间）
base_instant: Instant,
/// 基准时间点对应的UTC时间戳（微秒）
base_timestamp_us: i64,
/// 上次校准时间（用于检测是否需要重新校准）
last_calibration: Instant,
/// 校准间隔（秒）
calibration_interval_secs: u64,
}
impl HighPerformanceClock {
/// 创建新的高性能时钟
pub fn new() -> Self {
let base_instant = Instant::now();
let base_timestamp_us = chrono::Utc::now().timestamp_micros();
Self::new_with_calibration_interval(300) // 默认5分钟校准一次
}
Self { base_instant, base_timestamp_us }
/// 创建带自定义校准间隔的高性能时钟
pub fn new_with_calibration_interval(calibration_interval_secs: u64) -> Self {
// 通过多次采样来减少初始化误差
let mut best_offset = i64::MAX;
let mut best_instant = Instant::now();
let mut best_timestamp = chrono::Utc::now().timestamp_micros();
// 进行3次采样，选择延迟最小的
for _ in 0..3 {
let instant_before = Instant::now();
let timestamp = chrono::Utc::now().timestamp_micros();
let instant_after = Instant::now();
let sample_latency = instant_after.duration_since(instant_before).as_nanos() as i64;
if sample_latency < best_offset {
best_offset = sample_latency;
best_instant = instant_before;
best_timestamp = timestamp;
}
}
Self {
base_instant: best_instant,
base_timestamp_us: best_timestamp,
last_calibration: best_instant,
calibration_interval_secs,
}
}
/// 获取当前时间戳（微秒），使用单调时钟计算，避免系统调用
@@ -54,11 +85,53 @@ impl HighPerformanceClock {
self.base_timestamp_us + elapsed.as_micros() as i64
}
/// 获取高精度当前时间戳（微秒），在必要时进行校准
pub fn now_micros_with_calibration(&mut self) -> i64 {
// 检查是否需要重新校准
if self.last_calibration.elapsed().as_secs() >= self.calibration_interval_secs {
self.recalibrate();
}
self.now_micros()
}
/// 重新校准时钟，减少累积漂移
fn recalibrate(&mut self) {
let current_monotonic = Instant::now();
let current_utc = chrono::Utc::now().timestamp_micros();
// 计算预期的UTC时间戳（基于单调时钟）
let expected_utc = self.base_timestamp_us
+ current_monotonic.duration_since(self.base_instant).as_micros() as i64;
// 计算漂移量
let drift_us = current_utc - expected_utc;
// 如果漂移超过1毫秒，进行校准
if drift_us.abs() > 1000 {
self.base_instant = current_monotonic;
self.base_timestamp_us = current_utc;
}
self.last_calibration = current_monotonic;
}
/// 计算从指定时间戳到现在的消耗时间（微秒）
#[inline(always)]
pub fn elapsed_micros_since(&self, start_timestamp_us: i64) -> i64 {
self.now_micros() - start_timestamp_us
}
/// 获取高精度纳秒时间戳
#[inline(always)]
pub fn now_nanos(&self) -> i128 {
let elapsed = self.base_instant.elapsed();
(self.base_timestamp_us as i128 * 1000) + elapsed.as_nanos() as i128
}
/// 重置时钟（强制重新初始化）
pub fn reset(&mut self) {
*self = Self::new_with_calibration_interval(self.calibration_interval_secs);
}
}
impl Default for HighPerformanceClock {
@@ -67,14 +140,21 @@ impl Default for HighPerformanceClock {
}
}
/// 全局高性能时钟实例（使用OnceCell避免重复初始化）
/// 全局高性能时钟实例
static HIGH_PERF_CLOCK: once_cell::sync::OnceCell<HighPerformanceClock> =
once_cell::sync::OnceCell::new();
/// 获取全局高性能时钟实例
/// 获取全局高性能时钟实例（最简单的实现）
#[inline(always)]
pub fn get_high_perf_clock() -> &'static HighPerformanceClock {
HIGH_PERF_CLOCK.get_or_init(HighPerformanceClock::new)
pub fn get_high_perf_clock() -> i64 {
let clock = HIGH_PERF_CLOCK.get_or_init(HighPerformanceClock::new);
clock.now_micros()
}
/// 计算从指定时间戳到现在的消耗时间（微秒）
#[inline(always)]
pub fn elapsed_micros_since(start_timestamp_us: i64) -> i64 {
get_high_perf_clock() - start_timestamp_us
}
/// 轻量级事件包装器，避免频繁的Box分配
@@ -147,7 +227,7 @@ pub trait UnifiedEvent: Debug + Send + Sync {
fn event_type(&self) -> EventType;
/// Get transaction signature
fn signature(&self) -> &str;
fn signature(&self) -> &Signature;
/// Get slot number
fn slot(&self) -> u64;
@@ -535,7 +615,7 @@ pub trait EventParser: Send + Sync {
let slot = transaction.slot;
let block_time = transaction.block_time.map(|t| Timestamp { seconds: t as i64, nanos: 0 });
let program_received_time_us = chrono::Utc::now().timestamp_micros();
let program_received_time_us = get_high_perf_clock();
let bot_wallet = None;
let transaction_index = None;
// 解析指令事件
@@ -714,11 +794,10 @@ impl GenericEventParser {
transaction_index: Option<u64>,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(parser) = config.inner_instruction_parser {
let signature_str = Cow::Owned(signature.to_string());
let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 });
let block_time_ms = timestamp.seconds * 1000 + (timestamp.nanos as i64) / 1_000_000;
let metadata = EventMetadata::new(
signature_str,
signature,
slot,
timestamp.seconds,
block_time_ms,
@@ -752,11 +831,10 @@ impl GenericEventParser {
transaction_index: Option<u64>,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(parser) = config.instruction_parser {
let signature_str = Cow::Owned(signature.to_string());
let timestamp = block_time.unwrap_or(Timestamp { seconds: 0, nanos: 0 });
let block_time_ms = timestamp.seconds * 1000 + (timestamp.nanos as i64) / 1_000_000;
let metadata = EventMetadata::new(
signature_str,
signature,
slot,
timestamp.seconds,
block_time_ms,
@@ -941,9 +1019,9 @@ impl EventParser for GenericEventParser {
event.merge(&*inner_instruction_event);
}
// 设置处理时间（使用高性能时钟）
event.set_program_handle_time_consuming_us(
get_high_perf_clock().elapsed_micros_since(program_received_time_us),
);
event.set_program_handle_time_consuming_us(elapsed_micros_since(
program_received_time_us,
));
event = process_event(event, bot_wallet);
callback(&event);
}