gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-17 02:48:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: implement SIMD-accelerated event processing and optimize streaming performance

Browse Source 
- Add SIMD utilities for fast byte array comparison and discriminator matching
- Optimize event processor with batch processing and memory pool
- Refactor global state management with concurrent data structures
- Remove deprecated batch processing module
- Enhance metrics collection with reduced overhead
- Improve parser efficiency across all protocol implementations
- Add performance benchmarking dependencies (criterion, wide)
- Update documentation and examples for new architecture

Performance improvements:
- SIMD-accelerated byte operations for instruction parsing
- Concurrent HashMap (DashMap) for better multi-threading
- Optimized memory allocation patterns
- Reduced lock contention in event processing pipeline

Breaking changes: Removed batch.rs module, updated parser interfaces
This commit is contained in:
ysq
2025-08-31 22:18:18 +08:00
parent 52a489ae80
commit 74781e5cbe
30 changed files with 1297 additions and 1259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/event_parser/protocols/block/block_meta_event.rs
+3 -2
View File
@@ -1,3 +1,5 @@
use std::borrow::Cow;
use crate::impl_unified_event;
use crate::streaming::event_parser::common::{types::EventType, EventMetadata};
use borsh::BorshDeserialize;
@@ -20,8 +22,7 @@ impl BlockMetaEvent {
program_received_time_us: i64,
) -> Self {
let metadata = EventMetadata::new(
format!("block_{}_{}", slot, block_hash),
"".to_string(),
Cow::Borrowed(""),
slot,
block_time_ms / 1000,
block_time_ms,
src/streaming/event_parser/protocols/bonk/parser.rs
-28
View File
@@ -118,8 +118,6 @@ impl BonkEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = bonk_pool_create_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkPoolCreateEvent { metadata, ..event }))
} else {
None
@@ -132,8 +130,6 @@ impl BonkEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = bonk_trade_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, event.pool_state));
if metadata.event_type == EventType::BonkBuyExactIn
|| metadata.event_type == EventType::BonkBuyExactOut
{
@@ -166,9 +162,6 @@ impl BonkEventParser {
let minimum_amount_out = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_in,
@@ -207,9 +200,6 @@ impl BonkEventParser {
let maximum_amount_in = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_out,
@@ -248,9 +238,6 @@ impl BonkEventParser {
let minimum_amount_out = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_in,
@@ -289,9 +276,6 @@ impl BonkEventParser {
let maximum_amount_in = read_u64_le(data, 8)?;
let share_fee_rate = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!("{}-{}", metadata.signature, accounts[4]));
Some(Box::new(BonkTradeEvent {
metadata,
amount_out,
@@ -332,9 +316,6 @@ impl BonkEventParser {
let curve_param = Self::parse_curve_params(data, &mut offset)?;
let vesting_param = Self::parse_vesting_params(data, &mut offset)?;
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkPoolCreateEvent {
metadata,
payer: accounts[0],
@@ -369,9 +350,6 @@ impl BonkEventParser {
let vesting_param = Self::parse_vesting_params(data, &mut offset)?;
let amm_fee_on = data[offset];
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkPoolCreateEvent {
metadata,
payer: accounts[0],
@@ -514,9 +492,6 @@ impl BonkEventParser {
let quote_lot_size = u64::from_le_bytes(data[8..16].try_into().unwrap());
let market_vault_signer_nonce = data[16];
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkMigrateToAmmEvent {
metadata,
base_lot_size,
@@ -564,9 +539,6 @@ impl BonkEventParser {
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
let mut metadata = metadata;
metadata.set_id(metadata.signature.to_string());
Some(Box::new(BonkMigrateToCpswapEvent {
metadata,
payer: accounts[0],
src/streaming/event_parser/protocols/pumpfun/parser.rs
-22
View File
@@ -81,8 +81,6 @@ impl PumpFunEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pumpfun_migrate_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, event.user, event.mint));
Some(Box::new(PumpFunMigrateEvent { metadata, ..event }))
} else {
None
@@ -95,11 +93,6 @@ impl PumpFunEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pumpfun_create_token_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.name, event.symbol, event.mint
));
Some(Box::new(PumpFunCreateTokenEvent { metadata, ..event }))
} else {
None
@@ -112,11 +105,6 @@ impl PumpFunEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pumpfun_trade_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.mint, event.user, event.is_buy
));
Some(Box::new(PumpFunTradeEvent { metadata, ..event }))
} else {
None
@@ -151,9 +139,6 @@ impl PumpFunEventParser {
Pubkey::default()
};
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, name, symbol, accounts[0]));
Some(Box::new(PumpFunCreateTokenEvent {
metadata,
name: name.to_string(),
@@ -180,8 +165,6 @@ impl PumpFunEventParser {
}
let amount = u64::from_le_bytes(data[0..8].try_into().unwrap());
let max_sol_cost = u64::from_le_bytes(data[8..16].try_into().unwrap());
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], true));
Some(Box::new(PumpFunTradeEvent {
metadata,
global: accounts[0],
@@ -216,9 +199,6 @@ impl PumpFunEventParser {
}
let amount = u64::from_le_bytes(data[0..8].try_into().unwrap());
let min_sol_output = u64::from_le_bytes(data[8..16].try_into().unwrap());
let mut metadata = metadata;
metadata
.set_id(format!("{}-{}-{}-{}", metadata.signature, accounts[2], accounts[6], false));
Some(Box::new(PumpFunTradeEvent {
metadata,
global: accounts[0],
@@ -251,8 +231,6 @@ impl PumpFunEventParser {
if accounts.len() < 24 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[5], accounts[2]));
Some(Box::new(PumpFunMigrateEvent {
metadata,
global: accounts[0],
src/streaming/event_parser/protocols/pumpswap/parser.rs
-55
View File
@@ -91,11 +91,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_buy_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.user, event.pool, event.base_amount_out
));
Some(Box::new(PumpSwapBuyEvent { metadata, ..event }))
} else {
None
@@ -108,11 +103,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_sell_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.user, event.pool, event.base_amount_in
));
Some(Box::new(PumpSwapSellEvent { metadata, ..event }))
} else {
None
@@ -125,11 +115,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_create_pool_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.creator, event.base_amount_in
));
Some(Box::new(PumpSwapCreatePoolEvent { metadata, ..event }))
} else {
None
@@ -142,11 +127,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_deposit_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.user, event.lp_token_amount_out
));
Some(Box::new(PumpSwapDepositEvent { metadata, ..event }))
} else {
None
@@ -159,11 +139,6 @@ impl PumpSwapEventParser {
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(event) = pump_swap_withdraw_event_log_decode(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.user, event.lp_token_amount_in
));
Some(Box::new(PumpSwapWithdrawEvent { metadata, ..event }))
} else {
None
@@ -183,12 +158,6 @@ impl PumpSwapEventParser {
let base_amount_out = read_u64_le(data, 0)?;
let max_quote_amount_in = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[1], accounts[0], base_amount_out
));
Some(Box::new(PumpSwapBuyEvent {
metadata,
base_amount_out,
@@ -224,12 +193,6 @@ impl PumpSwapEventParser {
let base_amount_in = read_u64_le(data, 0)?;
let min_quote_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[1], accounts[0], base_amount_in
));
Some(Box::new(PumpSwapSellEvent {
metadata,
base_amount_in,
@@ -271,12 +234,6 @@ impl PumpSwapEventParser {
Pubkey::default()
};
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], base_amount_in
));
Some(Box::new(PumpSwapCreatePoolEvent {
metadata,
index,
@@ -311,12 +268,6 @@ impl PumpSwapEventParser {
let max_base_amount_in = u64::from_le_bytes(data[8..16].try_into().ok()?);
let max_quote_amount_in = u64::from_le_bytes(data[16..24].try_into().ok()?);
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], lp_token_amount_out
));
Some(Box::new(PumpSwapDepositEvent {
metadata,
lp_token_amount_out,
@@ -349,12 +300,6 @@ impl PumpSwapEventParser {
let min_base_amount_out = u64::from_le_bytes(data[8..16].try_into().ok()?);
let min_quote_amount_out = u64::from_le_bytes(data[16..24].try_into().ok()?);
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], lp_token_amount_in
));
Some(Box::new(PumpSwapWithdrawEvent {
metadata,
lp_token_amount_in,
src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs
-36
View File
@@ -102,12 +102,6 @@ impl RaydiumAmmV4EventParser {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4WithdrawPnlEvent {
metadata,
token_program: accounts[0],
@@ -141,12 +135,6 @@ impl RaydiumAmmV4EventParser {
}
let amount = read_u64_le(data, 0)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4WithdrawEvent {
metadata,
amount,
@@ -190,12 +178,6 @@ impl RaydiumAmmV4EventParser {
let init_pc_amount = read_u64_le(data, 9)?;
let init_coin_amount = read_u64_le(data, 17)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4Initialize2Event {
metadata,
nonce,
@@ -240,12 +222,6 @@ impl RaydiumAmmV4EventParser {
let max_pc_amount = read_u64_le(data, 8)?;
let base_side = read_u64_le(data, 16)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumAmmV4DepositEvent {
metadata,
max_coin_amount,
@@ -281,12 +257,6 @@ impl RaydiumAmmV4EventParser {
let max_amount_in = read_u64_le(data, 0)?;
let amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
let mut accounts = accounts.to_vec();
if accounts.len() == 17 {
// 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符
@@ -334,12 +304,6 @@ impl RaydiumAmmV4EventParser {
let amount_in = read_u64_le(data, 0)?;
let minimum_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
let mut accounts = accounts.to_vec();
if accounts.len() == 17 {
// 添加一个默认的 Pubkey 作为 amm_target_orders 的占位符
src/streaming/event_parser/protocols/raydium_clmm/parser.rs
-24
View File
@@ -124,8 +124,6 @@ impl RaydiumClmmEventParser {
if data.len() < 51 || accounts.len() < 22 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmOpenPositionV2Event {
metadata,
tick_lower_index: read_i32_le(data, 0)?,
@@ -172,8 +170,6 @@ impl RaydiumClmmEventParser {
if data.len() < 51 || accounts.len() < 20 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmOpenPositionWithToken22NftEvent {
metadata,
tick_lower_index: read_i32_le(data, 0)?,
@@ -217,8 +213,6 @@ impl RaydiumClmmEventParser {
if data.len() < 34 || accounts.len() < 15 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmIncreaseLiquidityV2Event {
metadata,
liquidity: read_u128_le(data, 0)?,
@@ -252,8 +246,6 @@ impl RaydiumClmmEventParser {
if data.len() < 24 || accounts.len() < 13 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmCreatePoolEvent {
metadata,
sqrt_price_x64: read_u128_le(data, 0)?,
@@ -283,8 +275,6 @@ impl RaydiumClmmEventParser {
if data.len() < 32 || accounts.len() < 16 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmDecreaseLiquidityV2Event {
metadata,
liquidity: read_u128_le(data, 0)?,
@@ -319,8 +309,6 @@ impl RaydiumClmmEventParser {
if accounts.len() < 6 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumClmmClosePositionEvent {
metadata,
nft_owner: accounts[0],
@@ -347,12 +335,6 @@ impl RaydiumClmmEventParser {
let sqrt_price_limit_x64 = read_u128_le(data, 16)?;
let is_base_input = read_u8_le(data, 32)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[2], accounts[3], accounts[4]
));
Some(Box::new(RaydiumClmmSwapEvent {
metadata,
amount,
@@ -387,12 +369,6 @@ impl RaydiumClmmEventParser {
let sqrt_price_limit_x64 = read_u128_le(data, 16)?;
let is_base_input = read_u8_le(data, 32)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[2], accounts[3], accounts[4]
));
Some(Box::new(RaydiumClmmSwapV2Event {
metadata,
amount,
src/streaming/event_parser/protocols/raydium_cpmm/parser.rs
-18
View File
@@ -92,8 +92,6 @@ impl RaydiumCpmmEventParser {
if data.len() < 24 || accounts.len() < 14 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumCpmmWithdrawEvent {
metadata,
lp_token_amount: read_u64_le(data, 0)?,
@@ -125,8 +123,6 @@ impl RaydiumCpmmEventParser {
if data.len() < 24 || accounts.len() < 20 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumCpmmInitializeEvent {
metadata,
init_amount0: read_u64_le(data, 0)?,
@@ -164,8 +160,6 @@ impl RaydiumCpmmEventParser {
if data.len() < 24 || accounts.len() < 13 {
return None;
}
let mut metadata = metadata;
metadata.set_id(format!("{}-{}-{}", metadata.signature, accounts[0], accounts[1]));
Some(Box::new(RaydiumCpmmDepositEvent {
metadata,
lp_token_amount: read_u64_le(data, 0)?,
@@ -200,12 +194,6 @@ impl RaydiumCpmmEventParser {
let amount_in = read_u64_le(data, 0)?;
let minimum_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumCpmmSwapEvent {
metadata,
amount_in,
@@ -239,12 +227,6 @@ impl RaydiumCpmmEventParser {
let max_amount_in = read_u64_le(data, 0)?;
let amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[3], accounts[10], accounts[11]
));
Some(Box::new(RaydiumCpmmSwapEvent {
metadata,
max_amount_in,